{ "schema_version": "0.1", "canitoolcall_version": "0.1.0.dev0", "engine": { "name": "sglang", "version": "0.5.20", "commit": null, "details": { "install": "cp312 manylinux wheel unpacked and added via .pth (scripts/engines/sglang.sh)", "dist_info": "sglang-0.5.20.dist-info", "tool_call_parsers": 39, "reasoning_parsers": 31, "deps": { "transformers": "5.12.1", "tokenizers": "0.22.2", "torch": "2.13.0", "xgrammar": "0.2.1", "pydantic": "2.13.5", "partial-json-parser": "0.2.1.1.post7", "orjson": "3.12.0" }, "pinned_version": "0.5.20", "tokens_per_step": "many" } }, "run": { "started_at": "2026-09-26T05:19:05Z", "finished_at": "2026-09-26T05:22:38Z", "platform": "linux-x86_64", "python": "3.12.14", "fixtures_digest": "a1e7b55367e195e8aa6e2ef746b2cc141b44f37caad0887ff8f30b104616f836", "strategies": [ "one", "special", "token", "rand:1:8", "rand:2:8", "rand:3:8", "rand:4:8", "rand:5:8" ], "normalization": "soft-v1", "synthetic_strategies": [] }, "cases": [ { "fixture_id": "deepseek/vllm-v3-no-tool-calls", "family": "deepseek", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv3", "reasoning_parser": null, "tool_call_detector": "DeepSeekV3Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "deepseekv31", "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3-0324", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3-0324", "revision": "e9b33add76883f293d6bf61f6bd89b497e80e335", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "deepseek_v3" }, "hf_config_error": null, "chat_template_sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "deepseek/vllm-v3-single-call", "family": "deepseek", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Tokyo\", \"unit\": \"celsius\"}', got '{}'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Tokyo\", \"unit\": \"celsius\"}', got '{}'" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Tokyo\", \"unit\": \"celsius\"}', got '{}'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Tokyo\", \"unit\": \"celsius\"}', got '{}'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Tokyo\", \"unit\": \"celsius\"}', got '{}'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Tokyo\", \"unit\": \"celsius\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Tokyo\", \"unit\": \"celsius\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Tokyo\", \"unit\": \"celsius\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Tokyo\", \"unit\": \"celsius\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Tokyo\", \"unit\": \"celsius\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '{}', got '{\"city\": \"Tokyo\", \"unit\": \"celsius\"}' | rand:4:8 (fail): tool_calls[0].arguments: one '{}', got '{\"city\": \"Tokyo\", \"unit\": \"celsius\"}' | rand:5:8 (fail): tool_calls[0].arguments: one '{}', got '{\"city\": \"Tokyo\", \"unit\": \"celsius\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv3", "reasoning_parser": null, "tool_call_detector": "DeepSeekV3Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "deepseekv31", "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3-0324", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3-0324", "revision": "e9b33add76883f293d6bf61f6bd89b497e80e335", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "deepseek_v3" }, "hf_config_error": null, "chat_template_sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"celsius\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"celsius\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"celsius\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"celsius\"}" } ], "exception": null } } } }, { "fixture_id": "deepseek/vllm-v3-parallel-no-newline", "family": "deepseek", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'search_hotels'], got ['search_hotels']" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'search_hotels'], got ['search_hotels']" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Tokyo\", \"unit\": \"celsius\"}', got '{\"city\": \"Tokyo\", \"unit\": \"celsius\"}\\n```<｜tool▁call▁end｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>search_hotels{\"location\": \"Tokyo\", \"check_in\": \"2025-01-15\"}'; tool_calls[1].arguments: expected '{\"location\": \"Tokyo\", \"check_in\": \"2025-01-15\"}', got '{}'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls[1].arguments: expected '{\"location\": \"Tokyo\", \"check_in\": \"2025-01-15\"}', got '{}'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'search_hotels'], got ['get_weather']" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'search_hotels'], got ['search_hotels']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'search_hotels'], got ['search_hotels']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Tokyo\", \"unit\": \"celsius\"}', got '{\"city\": \"Tokyo\", \"unit\": \"celsius\"}\\n```<｜tool▁call▁end｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>search_hotels{\"location\": \"Tokyo\", \"check_in\": \"2025-01-15\"}'; tool_calls[1].arguments: nonstream '{\"location\": \"Tokyo\", \"check_in\": \"2025-01-15\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls[1].arguments: nonstream '{\"location\": \"Tokyo\", \"check_in\": \"2025-01-15\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'search_hotels'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls: one ['search_hotels'], got ['get_weather', 'search_hotels'] | rand:1:8 (fail): tool_calls: one ['search_hotels'], got ['get_weather', 'search_hotels'] | rand:2:8 (fail): tool_calls: one ['search_hotels'], got ['get_weather', 'search_hotels'] | rand:3:8 (fail): tool_calls: one ['search_hotels'], got ['get_weather'] | rand:4:8 (fail): tool_calls: one ['search_hotels'], got ['get_weather', 'search_hotels'] | rand:5:8 (fail): tool_calls: one ['search_hotels'], got ['get_weather', 'search_hotels']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments contains '<｜tool▁call▁begin｜>'; tool_calls[0].arguments contains '<｜tool▁call▁end｜>'; tool_calls[0].arguments contains '<｜tool▁sep｜>'" }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls[0].arguments contains '<｜tool▁call▁begin｜>'; tool_calls[0].arguments contains '<｜tool▁call▁end｜>'; tool_calls[0].arguments contains '<｜tool▁sep｜>'" }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Tokyo\", \"unit\": \"celsius\"}\\n```<｜tool▁call▁end｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>search_hotels{\"location\": \"Tokyo\", \"check_in\": \"2025-01-15\"}': Extra data: line 2 column 1 (char 37))" }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Tokyo\", \"unit\": \"celsius\"}\\n```<｜tool▁call▁end｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>search_hotels{\"location\": \"Tokyo\", \"check_in\": \"2025-01-15\"}': Extra data: line 2 column 1 (char 37))" }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "fail", "detail": "expected 2 calls ['get_weather', 'search_hotels'], got 1 ['search_hotels']" }, { "check": "parallel_order", "strategy": "special", "status": "fail", "detail": "expected 2 calls ['get_weather', 'search_hotels'], got 1 ['search_hotels']" }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "fail", "detail": "expected 2 calls ['get_weather', 'search_hotels'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv3", "reasoning_parser": null, "tool_call_detector": "DeepSeekV3Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "deepseekv31", "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3-0324", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3-0324", "revision": "e9b33add76883f293d6bf61f6bd89b497e80e335", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "deepseek_v3" }, "hf_config_error": null, "chat_template_sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"celsius\"}" }, { "name": "search_hotels", "arguments_raw": "{\"location\": \"Tokyo\", \"check_in\": \"2025-01-15\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "search_hotels", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "search_hotels", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"celsius\"}" }, { "name": "search_hotels", "arguments_raw": "{\"location\": \"Tokyo\", \"check_in\": \"2025-01-15\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"celsius\"}\n```<｜tool▁call▁end｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>search_hotels{\"location\": \"Tokyo\", \"check_in\": \"2025-01-15\"}" }, { "name": "search_hotels", "arguments_raw": "{}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"celsius\"}" }, { "name": "search_hotels", "arguments_raw": "{}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"celsius\"}\n```<｜tool▁call▁end｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>search_hotels{\"location\": \"Tokyo\", \"check_in\": \"2025-01-15\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"celsius\"}" }, { "name": "search_hotels", "arguments_raw": "{\"location\": \"Tokyo\", \"check_in\": \"2025-01-15\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"celsius\"}" }, { "name": "search_hotels", "arguments_raw": "{\"location\": \"Tokyo\", \"check_in\": \"2025-01-15\"}" } ], "exception": null } } } }, { "fixture_id": "deepseek/vllm-v3-various-data-types", "family": "deepseek", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"ne..., got '{}'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"ne..., got '{}'" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"ne..., got '{}'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"ne..., got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"ne..., got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"ne..., got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '{}', got '{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"ne... | rand:1:8 (fail): tool_calls[0].arguments: one '{}', got '{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"ne... | rand:2:8 (fail): tool_calls[0].arguments: one '{}', got '{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"ne... | rand:4:8 (fail): tool_calls[0].arguments: one '{}', got '{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"ne... | rand:5:8 (fail): tool_calls[0].arguments: one '{}', got '{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"ne..." }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv3", "reasoning_parser": null, "tool_call_detector": "DeepSeekV3Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "deepseekv31", "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3-0324", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3-0324", "revision": "e9b33add76883f293d6bf61f6bd89b497e80e335", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "deepseek_v3" }, "hf_config_error": null, "chat_template_sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "test_function", "arguments_raw": "{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"nested\": \"value\"}, \"empty_array\": [], \"empty_object\": {}}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "test_function", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "test_function", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "test_function", "arguments_raw": "{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"nested\": \"value\"}, \"empty_array\": [], \"empty_object\": {}}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "test_function", "arguments_raw": "{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"nested\": \"value\"}, \"empty_array\": [], \"empty_object\": {}}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "test_function", "arguments_raw": "{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"nested\": \"value\"}, \"empty_array\": [], \"empty_object\": {}}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "test_function", "arguments_raw": "{}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "test_function", "arguments_raw": "{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"nested\": \"value\"}, \"empty_array\": [], \"empty_object\": {}}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "test_function", "arguments_raw": "{\"string_field\": \"hello\", \"int_field\": 42, \"float_field\": 3.14, \"bool_field\": true, \"null_field\": null, \"array_field\": [\"a\", \"b\", \"c\"], \"object_field\": {\"nested\": \"value\"}, \"empty_array\": [], \"empty_object\": {}}" } ], "exception": null } } } }, { "fixture_id": "deepseek/vllm-v3-escaped-strings", "family": "deepseek", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"text\": \"He said \\\\\"hello\\\\\"\", \"path\": \"C:\\\\\\\\Users\\\\\\\\file\", \"newline\": \"line1\\\\nline2\"}', got '{}'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"text\": \"He said \\\\\"hello\\\\\"\", \"path\": \"C:\\\\\\\\Users\\\\\\\\file\", \"newline\": \"line1\\\\nline2\"}', got '{}'" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"text\": \"He said \\\\\"hello\\\\\"\", \"path\": \"C:\\\\\\\\Users\\\\\\\\file\", \"newline\": \"line1\\\\nline2\"}', got '{}'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"text\": \"He said \\\\\"hello\\\\\"\", \"path\": \"C:\\\\\\\\Users\\\\\\\\file\", \"newline\": \"line1\\\\nline2\"}', got '{}'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"text\": \"He said \\\\\"hello\\\\\"\", \"path\": \"C:\\\\\\\\Users\\\\\\\\file\", \"newline\": \"line1\\\\nline2\"}', got '{}'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"text\": \"He said \\\\\"hello\\\\\"\", \"path\": \"C:\\\\\\\\Users\\\\\\\\file\", \"newline\": \"line1\\\\nline2\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"He said \\\\\"hello\\\\\"\", \"path\": \"C:\\\\\\\\Users\\\\\\\\file\", \"newline\": \"line1\\\\nline2\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"He said \\\\\"hello\\\\\"\", \"path\": \"C:\\\\\\\\Users\\\\\\\\file\", \"newline\": \"line1\\\\nline2\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"He said \\\\\"hello\\\\\"\", \"path\": \"C:\\\\\\\\Users\\\\\\\\file\", \"newline\": \"line1\\\\nline2\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"He said \\\\\"hello\\\\\"\", \"path\": \"C:\\\\\\\\Users\\\\\\\\file\", \"newline\": \"line1\\\\nline2\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"He said \\\\\"hello\\\\\"\", \"path\": \"C:\\\\\\\\Users\\\\\\\\file\", \"newline\": \"line1\\\\nline2\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"He said \\\\\"hello\\\\\"\", \"path\": \"C:\\\\\\\\Users\\\\\\\\file\", \"newline\": \"line1\\\\nline2\"}', got '{}'" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '{}', got '{\"text\": \"He said \\\\\"hello\\\\\"\", \"path\": \"C:\\\\\\\\Users\\\\\\\\file\", \"newline\": \"line1\\\\nline2\"}' | rand:1:8 (fail): tool_calls[0].arguments: one '{}', got '{\"text\": \"He said \\\\\"hello\\\\\"\", \"path\": \"C:\\\\\\\\Users\\\\\\\\file\", \"newline\": \"line1\\\\nline2\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv3", "reasoning_parser": null, "tool_call_detector": "DeepSeekV3Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "deepseekv31", "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3-0324", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3-0324", "revision": "e9b33add76883f293d6bf61f6bd89b497e80e335", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "deepseek_v3" }, "hf_config_error": null, "chat_template_sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "send_message", "arguments_raw": "{\"text\": \"He said \\\"hello\\\"\", \"path\": \"C:\\\\Users\\\\file\", \"newline\": \"line1\\nline2\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "send_message", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "send_message", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "send_message", "arguments_raw": "{\"text\": \"He said \\\"hello\\\"\", \"path\": \"C:\\\\Users\\\\file\", \"newline\": \"line1\\nline2\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "send_message", "arguments_raw": "{\"text\": \"He said \\\"hello\\\"\", \"path\": \"C:\\\\Users\\\\file\", \"newline\": \"line1\\nline2\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "send_message", "arguments_raw": "{}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "send_message", "arguments_raw": "{}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "send_message", "arguments_raw": "{}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "send_message", "arguments_raw": "{}" } ], "exception": null } } } }, { "fixture_id": "deepseek/vllm-v31-text-before-call", "family": "deepseek", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected 'normal text', got None; tool_calls[0].arguments: expected '{\"x\": 1}', got '{}'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected 'normal text', got None; tool_calls[0].arguments: expected '{\"x\": 1}', got '{}'" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected 'normal text', got None" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected 'normal text', got 'normal'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected 'normal text', got None" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected 'normal text', got None" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected 'normal text', got None" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream 'normal text', got None; tool_calls[0].arguments: nonstream '{\"x\": 1}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream 'normal text', got None; tool_calls[0].arguments: nonstream '{\"x\": 1}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream 'normal text', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream 'normal text', got 'normal'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream 'normal text', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream 'normal text', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream 'normal text', got None" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content: one None, got 'normal text'; tool_calls[0].arguments: one '{}', got '{\"x\":1}' | rand:1:8 (fail): tool_calls[0].arguments: one '{}', got '{\"x\":1}' | rand:2:8 (fail): content: one None, got 'normal'; tool_calls[0].arguments: one '{}', got '{\"x\":1}' | rand:3:8 (fail): tool_calls[0].arguments: one '{}', got '{\"x\":1}' | rand:4:8 (fail): tool_calls[0].arguments: one '{}', got '{\"x\":1}' | rand:5:8 (fail): tool_calls[0].arguments: one '{}', got '{\"x\":1}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv31", "reasoning_parser": "deepseek-v3", "tool_call_detector": "DeepSeekV31Detector", "reasoning_detector": "_DeepSeekV3Detector", "auto_detected": { "tool_call_parser": "deepseekv31", "reasoning_parser": "deepseek-v3" }, "model": "deepseek-ai/DeepSeek-V3.1", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3.1", "revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "deepseek_v3" }, "hf_config_error": null, "chat_template_sha256": "45690185067ea25c073bff2d3e425a45ee2564d2bbf6cdf7d885e139d460b8e0", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128799 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "normal text", "reasoning_content": null, "tool_calls": [ { "name": "foo", "arguments_raw": "{\"x\": 1}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "foo", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "foo", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": "normal text", "reasoning_content": null, "tool_calls": [ { "name": "foo", "arguments_raw": "{\"x\":1}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "foo", "arguments_raw": "{\"x\":1}" } ], "exception": null }, "rand:2:8": { "content": "normal", "reasoning_content": null, "tool_calls": [ { "name": "foo", "arguments_raw": "{\"x\":1}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "foo", "arguments_raw": "{\"x\":1}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "foo", "arguments_raw": "{\"x\":1}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "foo", "arguments_raw": "{\"x\":1}" } ], "exception": null } } } }, { "fixture_id": "deepseek/vllm-v41-text-before-parallel-calls", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "SGLang 0.5.20 has no DeepSeek-V4.1 parser: V4.1's spaced DSML markers ('DSML' + U+FF5C + ' calls') appear nowhere in sglang/srt; the deepseekv4 detector only knows V4's 'DSML' + U+FF5C + 'tool_calls'" }, { "fixture_id": "deepseek/vllm-v41-reasoning-text-before-parallel-calls", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "SGLang 0.5.20 has no DeepSeek-V4.1 parser: V4.1's spaced DSML markers ('DSML' + U+FF5C + ' calls') appear nowhere in sglang/srt; the deepseekv4 detector only knows V4's 'DSML' + U+FF5C + 'tool_calls'" }, { "fixture_id": "deepseek/sglang-v32-text-before-empty-invoke", "family": "deepseek", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me get the current date for you.', got 'Let me get the current date for you.\\n\\n\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me get the current date for you.', got 'Let me get the current date for you.\\n\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Let me get the current date for you.', got 'Let me get the current date for you.\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Let me get the current date for you.', got 'Let me get the current date for you.\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one 'Let me get the current date for you.', got 'Let me get the current date for you.\\n\\n\\n' | rand:4:8 (soft_pass): content (whitespace only): one 'Let me get the current date for you.', got 'Let me get the current date for you.\\n\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv32", "reasoning_parser": "deepseek-v3", "tool_call_detector": "DeepSeekV32Detector", "reasoning_detector": "_DeepSeekV3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3.2", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3.2", "revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV32ForCausalLM" ], "model_type": "deepseek_v32" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv32", "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128799 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "Let me get the current date for you.", "reasoning_content": null, "tool_calls": [ { "name": "get_date", "arguments_raw": "{}" } ], "exception": null }, "streams": { "one": { "content": "Let me get the current date for you.", "reasoning_content": null, "tool_calls": [ { "name": "get_date", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": "Let me get the current date for you.", "reasoning_content": null, "tool_calls": [ { "name": "get_date", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": "Let me get the current date for you.\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "get_date", "arguments_raw": "{}" } ], "exception": null }, "rand:1:8": { "content": "Let me get the current date for you.", "reasoning_content": null, "tool_calls": [ { "name": "get_date", "arguments_raw": "{}" } ], "exception": null }, "rand:2:8": { "content": "Let me get the current date for you.", "reasoning_content": null, "tool_calls": [ { "name": "get_date", "arguments_raw": "{}" } ], "exception": null }, "rand:3:8": { "content": "Let me get the current date for you.", "reasoning_content": null, "tool_calls": [ { "name": "get_date", "arguments_raw": "{}" } ], "exception": null }, "rand:4:8": { "content": "Let me get the current date for you.\n\n", "reasoning_content": null, "tool_calls": [ { "name": "get_date", "arguments_raw": "{}" } ], "exception": null }, "rand:5:8": { "content": "Let me get the current date for you.", "reasoning_content": null, "tool_calls": [ { "name": "get_date", "arguments_raw": "{}" } ], "exception": null } } } }, { "fixture_id": "deepseek/sglang-v4-self-closing-invoke", "family": "deepseek", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one None, got '\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv4", "reasoning_parser": "deepseek-v4", "tool_call_detector": "DeepSeekV4Detector", "reasoning_detector": "DeepSeekV4Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V4-Flash", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V4-Flash", "revision": "60d8d70770c6776ff598c94bb586a859a38244f1", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV4ForCausalLM" ], "model_type": "deepseek_v4" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv4", "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128822 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "submit", "arguments_raw": "{}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "submit", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "submit", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "submit", "arguments_raw": "{}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "submit", "arguments_raw": "{}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "submit", "arguments_raw": "{}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "submit", "arguments_raw": "{}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "submit", "arguments_raw": "{}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "submit", "arguments_raw": "{}" } ], "exception": null } } } }, { "fixture_id": "deepseek/vllm-v3-malformed-missing-brace", "family": "deepseek", "status": "fail", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for The arguments JSON is missing its closing brace; no valid call exists." }, { "check": "expected_error", "strategy": "special", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for The arguments JSON is missing its closing brace; no valid call exists." }, { "check": "expected_error", "strategy": "token", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for The arguments JSON is missing its closing brace; no valid call exists." }, { "check": "expected_error", "strategy": "rand:1:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for The arguments JSON is missing its closing brace; no valid call exists." }, { "check": "expected_error", "strategy": "rand:2:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for The arguments JSON is missing its closing brace; no valid call exists." }, { "check": "expected_error", "strategy": "rand:3:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for The arguments JSON is missing its closing brace; no valid call exists." }, { "check": "expected_error", "strategy": "rand:4:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for The arguments JSON is missing its closing brace; no valid call exists." }, { "check": "expected_error", "strategy": "rand:5:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for The arguments JSON is missing its closing brace; no valid call exists." }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_weather\\n```json\\n{\"city\": \"Tokyo\"\\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_weather\\n```json\\n{\"city\": \"Tokyo\"\\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_weather\\n```json\\n{\"city\": \"Tokyo\"\\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_weather\\n```json\\n{\"city\": \"Tokyo\"\\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_weather\\n```json\\n{\"city\": \"Tokyo\"\\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_weather\\n```json\\n{\"city\": \"Tokyo\"\\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_weather\\n```json\\n{\"city\": \"Tokyo\"\\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_weather\\n```json\\n{\"city\": \"Tokyo\"\\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '{}', got '{\"city\": \"Tokyo\"' | rand:2:8 (fail): tool_calls[0].arguments: one '{}', got '{\"city\": \"Tokyo\"' | rand:4:8 (fail): tool_calls[0].arguments: one '{}', got '{\"city\": \"Tokyo\"'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Tokyo\"': Expecting ',' delimiter: line 1 column 17 (char 16))" }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Tokyo\"': Expecting ',' delimiter: line 1 column 17 (char 16))" }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Tokyo\"': Expecting ',' delimiter: line 1 column 17 (char 16))" }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv3", "reasoning_parser": null, "tool_call_detector": "DeepSeekV3Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "deepseekv31", "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3-0324", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3-0324", "revision": "e9b33add76883f293d6bf61f6bd89b497e80e335", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "deepseek_v3" }, "hf_config_error": null, "chat_template_sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_weather\n```json\n{\"city\": \"Tokyo\"\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{}" } ], "exception": null } } } }, { "fixture_id": "deepseek/vllm-v3-malformed-missing-call-tokens", "family": "deepseek", "status": "pass", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv3", "reasoning_parser": null, "tool_call_detector": "DeepSeekV3Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "deepseekv31", "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3-0324", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3-0324", "revision": "e9b33add76883f293d6bf61f6bd89b497e80e335", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "deepseek_v3" }, "hf_config_error": null, "chat_template_sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "deepseek/v3-single-call", "family": "deepseek", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Paris\", \"unit\": \"c\"}', got '{}'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Paris\", \"unit\": \"c\"}', got '{}'" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Paris\", \"unit\": \"c\"}', got '{}'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Paris\", \"unit\": \"c\"}', got '{}'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Paris\", \"unit\": \"c\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Paris\", \"unit\": \"c\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Paris\", \"unit\": \"c\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Paris\", \"unit\": \"c\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '{}', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:2:8 (fail): tool_calls[0].arguments: one '{}', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:3:8 (fail): tool_calls[0].arguments: one '{}', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:5:8 (fail): tool_calls[0].arguments: one '{}', got '{\"city\": \"Paris\", \"unit\": \"c\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: $: 'city' is a required property" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: $: 'city' is a required property" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "fail", "detail": "[0] get_weather: $: 'city' is a required property" }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[0] get_weather: $: 'city' is a required property" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv3", "reasoning_parser": null, "tool_call_detector": "DeepSeekV3Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "deepseekv31", "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3-0324", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3-0324", "revision": "e9b33add76883f293d6bf61f6bd89b497e80e335", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "deepseek_v3" }, "hf_config_error": null, "chat_template_sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v3-parallel-two-calls", "family": "deepseek", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'search'], got ['search']" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'search'], got ['search']" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'search'], got ['get_weather']" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Zürich\", \"unit\": \"c\"}', got '{\"city\": \"Zürich\", \"unit\": \"c\"}\\n```<｜tool▁call▁end｜>\\n<｜tool▁call▁begin｜>function<｜tool▁sep｜>search{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"...; tool_calls[1].arguments: expected '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}', got '{}'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls[1].arguments: expected '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'search'], got ['search']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'search'], got ['search']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'search'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Zürich\", \"unit\": \"c\"}', got '{\"city\": \"Zürich\", \"unit\": \"c\"}\\n```<｜tool▁call▁end｜>\\n<｜tool▁call▁begin｜>function<｜tool▁sep｜>search{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"...; tool_calls[1].arguments: nonstream '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls[1].arguments: nonstream '{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}', got '{}'" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content (whitespace only): one None, got '\\n'; tool_calls: one ['search'], got ['get_weather', 'search'] | rand:1:8 (fail): tool_calls: one ['search'], got ['get_weather'] | rand:2:8 (fail): tool_calls: one ['search'], got ['get_weather', 'search'] | rand:3:8 (fail): tool_calls: one ['search'], got ['get_weather', 'search'] | rand:4:8 (fail): tool_calls: one ['search'], got ['get_weather', 'search'] | rand:5:8 (fail): tool_calls: one ['search'], got ['get_weather', 'search']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments contains '<｜tool▁call▁begin｜>'; tool_calls[0].arguments contains '<｜tool▁call▁end｜>'; tool_calls[0].arguments contains '<｜tool▁sep｜>'" }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls[0].arguments contains '<｜tool▁call▁begin｜>'; tool_calls[0].arguments contains '<｜tool▁call▁end｜>'; tool_calls[0].arguments contains '<｜tool▁sep｜>'" }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Zürich\", \"unit\": \"c\"}\\n```<｜tool▁call▁end｜>\\n<｜tool▁call▁begin｜>function<｜tool▁sep｜>search{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"...: Extra data: line 2 column 1 (char 32))" }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Zürich\", \"unit\": \"c\"}\\n```<｜tool▁call▁end｜>\\n<｜tool▁call▁begin｜>function<｜tool▁sep｜>search{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"...: Extra data: line 2 column 1 (char 32))" }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] search: $: 'query' is a required property" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] search: $: 'query' is a required property" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated; [1] search: $: 'query' is a required property" }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "fail", "detail": "[1] search: $: 'query' is a required property" }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "fail", "detail": "expected 2 calls ['get_weather', 'search'], got 1 ['search']" }, { "check": "parallel_order", "strategy": "special", "status": "fail", "detail": "expected 2 calls ['get_weather', 'search'], got 1 ['search']" }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "fail", "detail": "expected 2 calls ['get_weather', 'search'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv3", "reasoning_parser": null, "tool_call_detector": "DeepSeekV3Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "deepseekv31", "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3-0324", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3-0324", "revision": "e9b33add76883f293d6bf61f6bd89b497e80e335", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "deepseek_v3" }, "hf_config_error": null, "chat_template_sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "search", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "search", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}\n```<｜tool▁call▁end｜>\n<｜tool▁call▁begin｜>function<｜tool▁sep｜>search{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}\n```<｜tool▁call▁end｜>\n<｜tool▁call▁begin｜>function<｜tool▁sep｜>search{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" }, { "name": "search", "arguments_raw": "{}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v3-no-call", "family": "deepseek", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv3", "reasoning_parser": null, "tool_call_detector": "DeepSeekV3Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "deepseekv31", "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3-0324", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3-0324", "revision": "e9b33add76883f293d6bf61f6bd89b497e80e335", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "deepseek_v3" }, "hf_config_error": null, "chat_template_sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "deepseek/v3-text-before-call", "family": "deepseek", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got None; tool_calls[0].arguments: expected '{\"city\": \"London\"}', got '{}'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got None; tool_calls[0].arguments: expected '{\"city\": \"London\"}', got '{}'" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got 'Let me check the current'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got 'Let me check the current'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in'; tool_calls[0].arguments: expected '{\"city\": \"London\"}', got '{}'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"London\"}', got '{}'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got 'Let me check the current'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got None; tool_calls[0].arguments: nonstream '{\"city\": \"London\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got None; tool_calls[0].arguments: nonstream '{\"city\": \"London\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got 'Let me check the current'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got 'Let me check the current'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got 'Let me check the current conditions in'; tool_calls[0].arguments: nonstream '{\"city\": \"London\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"London\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got 'Let me check the current'" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content: one None, got 'Let me check the current conditions in London.'; tool_calls[0].arguments: one '{}', got '{\"city\": \"London\"}' | rand:1:8 (fail): content: one None, got 'Let me check the current'; tool_calls[0].arguments: one '{}', got '{\"city\": \"London\"}' | rand:2:8 (fail): content: one None, got 'Let me check the current'; tool_calls[0].arguments: one '{}', got '{\"city\": \"London\"}' | rand:3:8 (fail): content: one None, got 'Let me check the current conditions in' | rand:4:8 (fail): content: one None, got 'Let me check the current conditions in London.' | rand:5:8 (fail): content: one None, got 'Let me check the current'; tool_calls[0].arguments: one '{}', got '{\"city\": \"London\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: $: 'city' is a required property" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: $: 'city' is a required property" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "fail", "detail": "[0] get_weather: $: 'city' is a required property" }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[0] get_weather: $: 'city' is a required property" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv3", "reasoning_parser": null, "tool_call_detector": "DeepSeekV3Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "deepseekv31", "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3-0324", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3-0324", "revision": "e9b33add76883f293d6bf61f6bd89b497e80e335", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "deepseek_v3" }, "hf_config_error": null, "chat_template_sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "Let me check the current conditions in London.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": "Let me check the current conditions in London.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:1:8": { "content": "Let me check the current", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:2:8": { "content": "Let me check the current", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:3:8": { "content": "Let me check the current conditions in", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{}" } ], "exception": null }, "rand:4:8": { "content": "Let me check the current conditions in London.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{}" } ], "exception": null }, "rand:5:8": { "content": "Let me check the current", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v3-empty-arguments", "family": "deepseek", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv3", "reasoning_parser": null, "tool_call_detector": "DeepSeekV3Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "deepseekv31", "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3-0324", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3-0324", "revision": "e9b33add76883f293d6bf61f6bd89b497e80e335", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "deepseek_v3" }, "hf_config_error": null, "chat_template_sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "deepseek/v3-numeric-arguments", "family": "deepseek", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}', got '{}'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}', got '{}'" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}', got '{}'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}', got '{}'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}', got '{}'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}', got '{}'" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '{}', got '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}' | rand:3:8 (fail): tool_calls[0].arguments: one '{}', got '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] calculate: $: 'x' is a required property; $: 'y' is a required property" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] calculate: $: 'x' is a required property; $: 'y' is a required property" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "fail", "detail": "[0] calculate: $: 'x' is a required property; $: 'y' is a required property" }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "fail", "detail": "[0] calculate: $: 'x' is a required property; $: 'y' is a required property" }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[0] calculate: $: 'x' is a required property; $: 'y' is a required property" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "fail", "detail": "[0] calculate: $: 'x' is a required property; $: 'y' is a required property" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv3", "reasoning_parser": null, "tool_call_detector": "DeepSeekV3Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "deepseekv31", "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3-0324", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3-0324", "revision": "e9b33add76883f293d6bf61f6bd89b497e80e335", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "deepseek_v3" }, "hf_config_error": null, "chat_template_sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v3-unicode-emoji", "family": "deepseek", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}', got '{}'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}', got '{}'" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}', got '{}'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}', got '{}'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}', got '{}'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}', got '{}'" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '{}', got '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}' | rand:2:8 (fail): tool_calls[0].arguments: one '{}', got '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] translate: $: 'text' is a required property" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] translate: $: 'text' is a required property" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "fail", "detail": "[0] translate: $: 'text' is a required property" }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "fail", "detail": "[0] translate: $: 'text' is a required property" }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[0] translate: $: 'text' is a required property" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "fail", "detail": "[0] translate: $: 'text' is a required property" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv3", "reasoning_parser": null, "tool_call_detector": "DeepSeekV3Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "deepseekv31", "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3-0324", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3-0324", "revision": "e9b33add76883f293d6bf61f6bd89b497e80e335", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "deepseek_v3" }, "hf_config_error": null, "chat_template_sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v3-marker-in-arguments", "family": "deepseek", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"text\": \"Reply as ```json\\\\n{\\\\\"a\\\\\": 1}\\\\n``` or use <｜tool▁sep｜> tokens.\"}', got '{}'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"text\": \"Reply as ```json\\\\n{\\\\\"a\\\\\": 1}\\\\n``` or use <｜tool▁sep｜> tokens.\"}', got '{}'" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"text\": \"Reply as ```json\\\\n{\\\\\"a\\\\\": 1}\\\\n``` or use <｜tool▁sep｜> tokens.\"}', got '{}'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"text\": \"Reply as ```json\\\\n{\\\\\"a\\\\\": 1}\\\\n``` or use <｜tool▁sep｜> tokens.\"}', got '{}'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"text\": \"Reply as ```json\\\\n{\\\\\"a\\\\\": 1}\\\\n``` or use <｜tool▁sep｜> tokens.\"}', got '{}'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"Reply as ```json\\\\n{\\\\\"a\\\\\": 1}\\\\n``` or use <｜tool▁sep｜> tokens.\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"Reply as ```json\\\\n{\\\\\"a\\\\\": 1}\\\\n``` or use <｜tool▁sep｜> tokens.\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"Reply as ```json\\\\n{\\\\\"a\\\\\": 1}\\\\n``` or use <｜tool▁sep｜> tokens.\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"Reply as ```json\\\\n{\\\\\"a\\\\\": 1}\\\\n``` or use <｜tool▁sep｜> tokens.\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"Reply as ```json\\\\n{\\\\\"a\\\\\": 1}\\\\n``` or use <｜tool▁sep｜> tokens.\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '{}', got '{\"text\": \"Reply as ```json\\\\n{\\\\\"a\\\\\": 1}\\\\n``` or use <｜tool▁sep｜> tokens.\"}' | rand:1:8 (fail): tool_calls[0].arguments: one '{}', got '{\"text\": \"Reply as ```json\\\\n{\\\\\"a\\\\\": 1}\\\\n``` or use <｜tool▁sep｜> tokens.\"}' | rand:5:8 (fail): tool_calls[0].arguments: one '{}', got '{\"text\": \"Reply as ```json\\\\n{\\\\\"a\\\\\": 1}\\\\n``` or use <｜tool▁sep｜> tokens.\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] echo: $: 'text' is a required property" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] echo: $: 'text' is a required property" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "fail", "detail": "[0] echo: $: 'text' is a required property" }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "fail", "detail": "[0] echo: $: 'text' is a required property" }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[0] echo: $: 'text' is a required property" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv3", "reasoning_parser": null, "tool_call_detector": "DeepSeekV3Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "deepseekv31", "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3-0324", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3-0324", "revision": "e9b33add76883f293d6bf61f6bd89b497e80e335", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "deepseek_v3" }, "hf_config_error": null, "chat_template_sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Reply as ```json\\n{\\\"a\\\": 1}\\n``` or use <｜tool▁sep｜> tokens.\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Reply as ```json\\n{\\\"a\\\": 1}\\n``` or use <｜tool▁sep｜> tokens.\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Reply as ```json\\n{\\\"a\\\": 1}\\n``` or use <｜tool▁sep｜> tokens.\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Reply as ```json\\n{\\\"a\\\": 1}\\n``` or use <｜tool▁sep｜> tokens.\"}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v3-truncated-mid-arguments", "family": "deepseek", "status": "pass", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv3", "reasoning_parser": null, "tool_call_detector": "DeepSeekV3Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "deepseekv31", "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3-0324", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3-0324", "revision": "e9b33add76883f293d6bf61f6bd89b497e80e335", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "deepseek_v3" }, "hf_config_error": null, "chat_template_sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "deepseek/v3-nested-json", "family": "deepseek", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam..., got '{}'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam..., got '{}'" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam..., got '{}'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam..., got '{}'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam..., got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam..., got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam..., got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam..., got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '{}', got '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam... | rand:1:8 (fail): tool_calls[0].arguments: one '{}', got '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam... | rand:3:8 (fail): tool_calls[0].arguments: one '{}', got '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam... | rand:5:8 (fail): tool_calls[0].arguments: one '{}', got '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam..." }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] create_event: $: 'title' is a required property; $: 'when' is a required property" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] create_event: $: 'title' is a required property; $: 'when' is a required property" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "fail", "detail": "[0] create_event: $: 'title' is a required property; $: 'when' is a required property" }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[0] create_event: $: 'title' is a required property; $: 'when' is a required property" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv3", "reasoning_parser": null, "tool_call_detector": "DeepSeekV3Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "deepseekv31", "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3-0324", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3-0324", "revision": "e9b33add76883f293d6bf61f6bd89b497e80e335", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "deepseek_v3" }, "hf_config_error": null, "chat_template_sha256": "22e97ba043dacf9df1f071a9de9d2bb6b0757491f16e96e810834b86e1641286", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v31-single-call", "family": "deepseek", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Paris\", \"unit\": \"c\"}', got '{}'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Paris\", \"unit\": \"c\"}', got '{}'" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Paris\", \"unit\": \"c\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Paris\", \"unit\": \"c\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '{}', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:1:8 (fail): tool_calls[0].arguments: one '{}', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:2:8 (fail): tool_calls[0].arguments: one '{}', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:3:8 (fail): tool_calls[0].arguments: one '{}', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:4:8 (fail): tool_calls[0].arguments: one '{}', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:5:8 (fail): tool_calls[0].arguments: one '{}', got '{\"city\": \"Paris\", \"unit\": \"c\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: $: 'city' is a required property" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: $: 'city' is a required property" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv31", "reasoning_parser": "deepseek-v3", "tool_call_detector": "DeepSeekV31Detector", "reasoning_detector": "_DeepSeekV3Detector", "auto_detected": { "tool_call_parser": "deepseekv31", "reasoning_parser": "deepseek-v3" }, "model": "deepseek-ai/DeepSeek-V3.1", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3.1", "revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "deepseek_v3" }, "hf_config_error": null, "chat_template_sha256": "45690185067ea25c073bff2d3e425a45ee2564d2bbf6cdf7d885e139d460b8e0", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128799 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v31-parallel-two-calls", "family": "deepseek", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'search'], got ['get_weather<｜tool▁sep｜>{\"city\": \"Zürich\", \"unit\": \"c\"}<｜tool▁call▁end｜><｜tool▁call▁begin｜>search']" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'search'], got ['get_weather<｜tool▁sep｜>{\"city\": \"Zürich\", \"unit\": \"c\"}<｜tool▁call▁end｜><｜tool▁call▁begin｜>search']" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'search'], got ['get_weather']" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'search'], got ['get_weather']" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'search'], got ['get_weather<｜tool▁sep｜>{\"city\": \"Zürich\", \"unit\": \"c\"}<｜tool▁call▁end｜><｜tool▁call▁begin｜>search']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'search'], got ['get_weather<｜tool▁sep｜>{\"city\": \"Zürich\", \"unit\": \"c\"}<｜tool▁call▁end｜><｜tool▁call▁begin｜>search']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'search'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'search'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls: one ['get_weather<｜tool▁sep｜>{\"city\": \"Zürich\", \"unit\": \"c\"}<｜tool▁call▁end｜><｜tool▁call▁begin｜>search'], got ['get_weather', 'search'] | rand:1:8 (fail): tool_calls: one ['get_weather<｜tool▁sep｜>{\"city\": \"Zürich\", \"unit\": \"c\"}<｜tool▁call▁end｜><｜tool▁call▁begin｜>search'], got ['get_weather', 'search'] | rand:2:8 (fail): tool_calls: one ['get_weather<｜tool▁sep｜>{\"city\": \"Zürich\", \"unit\": \"c\"}<｜tool▁call▁end｜><｜tool▁call▁begin｜>search'], got ['get_weather'] | rand:3:8 (fail): tool_calls: one ['get_weather<｜tool▁sep｜>{\"city\": \"Zürich\", \"unit\": \"c\"}<｜tool▁call▁end｜><｜tool▁call▁begin｜>search'], got ['get_weather', 'search'] | rand:4:8 (fail): tool_calls: one ['get_weather<｜tool▁sep｜>{\"city\": \"Zürich\", \"unit\": \"c\"}<｜tool▁call▁end｜><｜tool▁call▁begin｜>search'], got ['get_weather'] | rand:5:8 (fail): tool_calls: one ['get_weather<｜tool▁sep｜>{\"city\": \"Zürich\", \"unit\": \"c\"}<｜tool▁call▁end｜><｜tool▁call▁begin｜>search'], got ['get_weather', 'search']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "fail", "detail": "tool_calls[0].name contains '<｜tool▁call▁begin｜>'; tool_calls[0].name contains '<｜tool▁call▁end｜>'; tool_calls[0].name contains '<｜tool▁sep｜>'" }, { "check": "no_leakage", "strategy": "special", "status": "fail", "detail": "tool_calls[0].name contains '<｜tool▁call▁begin｜>'; tool_calls[0].name contains '<｜tool▁call▁end｜>'; tool_calls[0].name contains '<｜tool▁sep｜>'" }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Zürich\", \"unit\": \"c{\"query\": \"{\"query\": \"c{\"query\": \"café \\\\\"{\"query\": \"café \\\\\"best\\\\\"\", \"filters\":{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags...: Expecting ',' delimiter: line 1 column 32 (char 31))" }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Zürich\", \"unit\": \"{\"query{\"query\": \"c{\"query\": \"café{\"query\": \"café \\\\\"best{\"query\": \"café \\\\\"best\\\\\"{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tag...: Expecting ',' delimiter: line 1 column 31 (char 30))" }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] 'get_weather<｜tool▁sep｜>{\"city\": \"Zürich\", \"unit\": \"c\"}<｜tool▁call▁end｜><｜tool▁call▁begin｜>search' is not an offered tool" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] 'get_weather<｜tool▁sep｜>{\"city\": \"Zürich\", \"unit\": \"c\"}<｜tool▁call▁end｜><｜tool▁call▁begin｜>search' is not an offered tool" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "fail", "detail": "expected 2 calls ['get_weather', 'search'], got 1 ['get_weather<｜tool▁sep｜>{\"city\": \"Zürich\", \"unit\": \"c\"}<｜tool▁call▁end｜><｜tool▁call▁begin｜>search']" }, { "check": "parallel_order", "strategy": "special", "status": "fail", "detail": "expected 2 calls ['get_weather', 'search'], got 1 ['get_weather<｜tool▁sep｜>{\"city\": \"Zürich\", \"unit\": \"c\"}<｜tool▁call▁end｜><｜tool▁call▁begin｜>search']" }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "fail", "detail": "expected 2 calls ['get_weather', 'search'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "fail", "detail": "expected 2 calls ['get_weather', 'search'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv31", "reasoning_parser": "deepseek-v3", "tool_call_detector": "DeepSeekV31Detector", "reasoning_detector": "_DeepSeekV3Detector", "auto_detected": { "tool_call_parser": "deepseekv31", "reasoning_parser": "deepseek-v3" }, "model": "deepseek-ai/DeepSeek-V3.1", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3.1", "revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "deepseek_v3" }, "hf_config_error": null, "chat_template_sha256": "45690185067ea25c073bff2d3e425a45ee2564d2bbf6cdf7d885e139d460b8e0", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128799 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather<｜tool▁sep｜>{\"city\": \"Zürich\", \"unit\": \"c\"}<｜tool▁call▁end｜><｜tool▁call▁begin｜>search", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather<｜tool▁sep｜>{\"city\": \"Zürich\", \"unit\": \"c\"}<｜tool▁call▁end｜><｜tool▁call▁begin｜>search", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c{\"query\": \"{\"query\": \"c{\"query\": \"café \\\"{\"query\": \"café \\\"best\\\"\", \"filters\":{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\":{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"{\"query{\"query\": \"c{\"query\": \"café{\"query\": \"café \\\"best{\"query\": \"café \\\"best\\\"{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\":{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v31-no-call", "family": "deepseek", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv31", "reasoning_parser": "deepseek-v3", "tool_call_detector": "DeepSeekV31Detector", "reasoning_detector": "_DeepSeekV3Detector", "auto_detected": { "tool_call_parser": "deepseekv31", "reasoning_parser": "deepseek-v3" }, "model": "deepseek-ai/DeepSeek-V3.1", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3.1", "revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "deepseek_v3" }, "hf_config_error": null, "chat_template_sha256": "45690185067ea25c073bff2d3e425a45ee2564d2bbf6cdf7d885e139d460b8e0", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128799 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "deepseek/v31-text-before-call", "family": "deepseek", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got None; tool_calls[0].arguments: expected '{\"city\": \"London\"}', got '{}'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got None; tool_calls[0].arguments: expected '{\"city\": \"London\"}', got '{}'" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got 'Let me check the current'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got 'Let me check the current'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got 'Let me check the current'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got None; tool_calls[0].arguments: nonstream '{\"city\": \"London\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got None; tool_calls[0].arguments: nonstream '{\"city\": \"London\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got 'Let me check the current'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got 'Let me check the current'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got 'Let me check the current conditions in'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got 'Let me check the current'" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content: one None, got 'Let me check the current conditions in London.'; tool_calls[0].arguments: one '{}', got '{\"city\": \"London\"}' | rand:1:8 (fail): content: one None, got 'Let me check the current'; tool_calls[0].arguments: one '{}', got '{\"city\": \"London\"}' | rand:2:8 (fail): content: one None, got 'Let me check the current'; tool_calls[0].arguments: one '{}', got '{\"city\": \"London\"}' | rand:3:8 (fail): content: one None, got 'Let me check the current conditions in'; tool_calls[0].arguments: one '{}', got '{\"city\": \"London\"}' | rand:4:8 (fail): content: one None, got 'Let me check the current conditions in London.'; tool_calls[0].arguments: one '{}', got '{\"city\": \"London\"}' | rand:5:8 (fail): content: one None, got 'Let me check the current'; tool_calls[0].arguments: one '{}', got '{\"city\": \"London\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: $: 'city' is a required property" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: $: 'city' is a required property" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv31", "reasoning_parser": "deepseek-v3", "tool_call_detector": "DeepSeekV31Detector", "reasoning_detector": "_DeepSeekV3Detector", "auto_detected": { "tool_call_parser": "deepseekv31", "reasoning_parser": "deepseek-v3" }, "model": "deepseek-ai/DeepSeek-V3.1", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3.1", "revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "deepseek_v3" }, "hf_config_error": null, "chat_template_sha256": "45690185067ea25c073bff2d3e425a45ee2564d2bbf6cdf7d885e139d460b8e0", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128799 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "Let me check the current conditions in London.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": "Let me check the current conditions in London.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:1:8": { "content": "Let me check the current", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:2:8": { "content": "Let me check the current", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:3:8": { "content": "Let me check the current conditions in", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:4:8": { "content": "Let me check the current conditions in London.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:5:8": { "content": "Let me check the current", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v31-empty-arguments", "family": "deepseek", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv31", "reasoning_parser": "deepseek-v3", "tool_call_detector": "DeepSeekV31Detector", "reasoning_detector": "_DeepSeekV3Detector", "auto_detected": { "tool_call_parser": "deepseekv31", "reasoning_parser": "deepseek-v3" }, "model": "deepseek-ai/DeepSeek-V3.1", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3.1", "revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "deepseek_v3" }, "hf_config_error": null, "chat_template_sha256": "45690185067ea25c073bff2d3e425a45ee2564d2bbf6cdf7d885e139d460b8e0", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128799 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "deepseek/v31-numeric-arguments", "family": "deepseek", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}', got '{}'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}', got '{}'" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '{}', got '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}' | rand:1:8 (fail): tool_calls[0].arguments: one '{}', got '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}' | rand:2:8 (fail): tool_calls[0].arguments: one '{}', got '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}' | rand:3:8 (fail): tool_calls[0].arguments: one '{}', got '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}' | rand:4:8 (fail): tool_calls[0].arguments: one '{}', got '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}' | rand:5:8 (fail): tool_calls[0].arguments: one '{}', got '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] calculate: $: 'x' is a required property; $: 'y' is a required property" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] calculate: $: 'x' is a required property; $: 'y' is a required property" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv31", "reasoning_parser": "deepseek-v3", "tool_call_detector": "DeepSeekV31Detector", "reasoning_detector": "_DeepSeekV3Detector", "auto_detected": { "tool_call_parser": "deepseekv31", "reasoning_parser": "deepseek-v3" }, "model": "deepseek-ai/DeepSeek-V3.1", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3.1", "revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "deepseek_v3" }, "hf_config_error": null, "chat_template_sha256": "45690185067ea25c073bff2d3e425a45ee2564d2bbf6cdf7d885e139d460b8e0", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128799 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v31-unicode-emoji", "family": "deepseek", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}', got '{}'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}', got '{}'" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '{}', got '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}' | rand:1:8 (fail): tool_calls[0].arguments: one '{}', got '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}' | rand:2:8 (fail): tool_calls[0].arguments: one '{}', got '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}' | rand:3:8 (fail): tool_calls[0].arguments: one '{}', got '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}' | rand:4:8 (fail): tool_calls[0].arguments: one '{}', got '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}' | rand:5:8 (fail): tool_calls[0].arguments: one '{}', got '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] translate: $: 'text' is a required property" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] translate: $: 'text' is a required property" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv31", "reasoning_parser": "deepseek-v3", "tool_call_detector": "DeepSeekV31Detector", "reasoning_detector": "_DeepSeekV3Detector", "auto_detected": { "tool_call_parser": "deepseekv31", "reasoning_parser": "deepseek-v3" }, "model": "deepseek-ai/DeepSeek-V3.1", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3.1", "revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "deepseek_v3" }, "hf_config_error": null, "chat_template_sha256": "45690185067ea25c073bff2d3e425a45ee2564d2bbf6cdf7d885e139d460b8e0", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128799 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v31-marker-in-arguments", "family": "deepseek", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>echo<｜tool▁sep｜>{\"text\": \"Tokens like <｜tool▁call▁begin｜>f<｜tool▁sep｜>{} are markup.\"}<｜tool▁call▁end｜><｜tool▁calls▁e...; tool_calls: expected ['echo'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['echo'], got ['echo<｜tool▁sep｜>{\"text\": \"Tokens like <｜tool▁call▁begin｜>f']" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['echo'], got ['echo<｜tool▁sep｜>{\"text\": \"Tokens like <｜tool▁call▁begin｜>f']" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected None, got ' are markup.\"}'; tool_calls[0].arguments: expected '{\"text\": \"Tokens like <｜tool▁call▁begin｜>f<｜tool▁sep｜>{} are markup.\"}', got '{\"text\": \"Tokens like <｜tool▁call▁begin｜>f{}'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"text\": \"Tokens like <｜tool▁call▁begin｜>f<｜tool▁sep｜>{} are markup.\"}', got '{\"text\": \"Tokens like <｜tool▁call▁begin｜>{} are markup.\"}{} are markup.\"}'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"text\": \"Tokens like <｜tool▁call▁begin｜>f<｜tool▁sep｜>{} are markup.\"}', got '{\"text\": \"Tokens like{} are markup.\"}{} are markup.\"}'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"text\": \"Tokens like <｜tool▁call▁begin｜>f<｜tool▁sep｜>{} are markup.\"}', got '{\"text\": \"Tokens like <｜tool▁call▁begin｜>f{} are markup.\"}'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"text\": \"Tokens like <｜tool▁call▁begin｜>f<｜tool▁sep｜>{} are markup.\"}', got '{\"text\": \"Tokens like{} are markup.\"{} are markup.\"}'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"text\": \"Tokens like <｜tool▁call▁begin｜>f<｜tool▁sep｜>{} are markup.\"}', got '{\"text\": \"Tokens like <｜tool▁call▁begin｜>{} are markup.\"}{} are markup.\"}'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>echo<｜tool▁sep｜>{\"text\": \"Tokens like <｜tool▁call▁begin｜>f<｜tool▁sep｜>{} are markup.\"}<｜tool▁call▁end｜><｜tool▁calls▁e..., got None; tool_calls: nonstream [], got ['echo<｜tool▁sep｜>{\"text\": \"Tokens like <｜tool▁call▁begin｜>f']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>echo<｜tool▁sep｜>{\"text\": \"Tokens like <｜tool▁call▁begin｜>f<｜tool▁sep｜>{} are markup.\"}<｜tool▁call▁end｜><｜tool▁calls▁e..., got None; tool_calls: nonstream [], got ['echo<｜tool▁sep｜>{\"text\": \"Tokens like <｜tool▁call▁begin｜>f']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>echo<｜tool▁sep｜>{\"text\": \"Tokens like <｜tool▁call▁begin｜>f<｜tool▁sep｜>{} are markup.\"}<｜tool▁call▁end｜><｜tool▁calls▁e..., got ' are markup.\"}'; tool_calls: nonstream [], got ['echo']" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>echo<｜tool▁sep｜>{\"text\": \"Tokens like <｜tool▁call▁begin｜>f<｜tool▁sep｜>{} are markup.\"}<｜tool▁call▁end｜><｜tool▁calls▁e..., got None; tool_calls: nonstream [], got ['echo']" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>echo<｜tool▁sep｜>{\"text\": \"Tokens like <｜tool▁call▁begin｜>f<｜tool▁sep｜>{} are markup.\"}<｜tool▁call▁end｜><｜tool▁calls▁e..., got None; tool_calls: nonstream [], got ['echo']" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>echo<｜tool▁sep｜>{\"text\": \"Tokens like <｜tool▁call▁begin｜>f<｜tool▁sep｜>{} are markup.\"}<｜tool▁call▁end｜><｜tool▁calls▁e..., got None; tool_calls: nonstream [], got ['echo']" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>echo<｜tool▁sep｜>{\"text\": \"Tokens like <｜tool▁call▁begin｜>f<｜tool▁sep｜>{} are markup.\"}<｜tool▁call▁end｜><｜tool▁calls▁e..., got None; tool_calls: nonstream [], got ['echo']" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>echo<｜tool▁sep｜>{\"text\": \"Tokens like <｜tool▁call▁begin｜>f<｜tool▁sep｜>{} are markup.\"}<｜tool▁call▁end｜><｜tool▁calls▁e..., got None; tool_calls: nonstream [], got ['echo']" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content: one None, got ' are markup.\"}'; tool_calls: one ['echo<｜tool▁sep｜>{\"text\": \"Tokens like <｜tool▁call▁begin｜>f'], got ['echo'] | rand:1:8 (fail): tool_calls: one ['echo<｜tool▁sep｜>{\"text\": \"Tokens like <｜tool▁call▁begin｜>f'], got ['echo'] | rand:2:8 (fail): tool_calls: one ['echo<｜tool▁sep｜>{\"text\": \"Tokens like <｜tool▁call▁begin｜>f'], got ['echo'] | rand:3:8 (fail): tool_calls: one ['echo<｜tool▁sep｜>{\"text\": \"Tokens like <｜tool▁call▁begin｜>f'], got ['echo'] | rand:4:8 (fail): tool_calls: one ['echo<｜tool▁sep｜>{\"text\": \"Tokens like <｜tool▁call▁begin｜>f'], got ['echo'] | rand:5:8 (fail): tool_calls: one ['echo<｜tool▁sep｜>{\"text\": \"Tokens like <｜tool▁call▁begin｜>f'], got ['echo']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '<｜tool▁calls▁begin｜>'; content contains '<｜tool▁calls▁end｜>'; content contains '<｜tool▁call▁begin｜>'; content contains '<｜tool▁call▁end｜>'; content contains '<｜tool▁sep｜>'" }, { "check": "no_leakage", "strategy": "one", "status": "fail", "detail": "tool_calls[0].name contains '<｜tool▁call▁begin｜>'; tool_calls[0].name contains '<｜tool▁sep｜>'" }, { "check": "no_leakage", "strategy": "special", "status": "fail", "detail": "tool_calls[0].name contains '<｜tool▁call▁begin｜>'; tool_calls[0].name contains '<｜tool▁sep｜>'" }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "fail", "detail": "[0] echo: arguments are not valid JSON text ('{\"text\": \"Tokens like <｜tool▁call▁begin｜>f{}': Unterminated string starting at: line 1 column 10 (char 9))" }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "fail", "detail": "[0] echo: arguments are not valid JSON text ('{\"text\": \"Tokens like <｜tool▁call▁begin｜>{} are markup.\"}{} are markup.\"}': Extra data: line 1 column 58 (char 57))" }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "fail", "detail": "[0] echo: arguments are not valid JSON text ('{\"text\": \"Tokens like{} are markup.\"}{} are markup.\"}': Extra data: line 1 column 38 (char 37))" }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "fail", "detail": "[0] echo: arguments are not valid JSON text ('{\"text\": \"Tokens like{} are markup.\"{} are markup.\"}': Expecting ',' delimiter: line 1 column 37 (char 36))" }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "fail", "detail": "[0] echo: arguments are not valid JSON text ('{\"text\": \"Tokens like <｜tool▁call▁begin｜>{} are markup.\"}{} are markup.\"}': Extra data: line 1 column 58 (char 57))" }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] 'echo<｜tool▁sep｜>{\"text\": \"Tokens like <｜tool▁call▁begin｜>f' is not an offered tool" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] 'echo<｜tool▁sep｜>{\"text\": \"Tokens like <｜tool▁call▁begin｜>f' is not an offered tool" }, { "check": "arguments_schema", "strategy": "token", "status": "fail", "detail": "[0] echo: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "fail", "detail": "[0] echo: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "fail", "detail": "[0] echo: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[0] echo: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "fail", "detail": "[0] echo: arguments are not a JSON object; not validated" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv31", "reasoning_parser": "deepseek-v3", "tool_call_detector": "DeepSeekV31Detector", "reasoning_detector": "_DeepSeekV3Detector", "auto_detected": { "tool_call_parser": "deepseekv31", "reasoning_parser": "deepseek-v3" }, "model": "deepseek-ai/DeepSeek-V3.1", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3.1", "revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "deepseek_v3" }, "hf_config_error": null, "chat_template_sha256": "45690185067ea25c073bff2d3e425a45ee2564d2bbf6cdf7d885e139d460b8e0", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128799 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>echo<｜tool▁sep｜>{\"text\": \"Tokens like <｜tool▁call▁begin｜>f<｜tool▁sep｜>{} are markup.\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo<｜tool▁sep｜>{\"text\": \"Tokens like <｜tool▁call▁begin｜>f", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo<｜tool▁sep｜>{\"text\": \"Tokens like <｜tool▁call▁begin｜>f", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": " are markup.\"}", "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Tokens like <｜tool▁call▁begin｜>f{}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Tokens like <｜tool▁call▁begin｜>{} are markup.\"}{} are markup.\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Tokens like{} are markup.\"}{} are markup.\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Tokens like <｜tool▁call▁begin｜>f{} are markup.\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Tokens like{} are markup.\"{} are markup.\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Tokens like <｜tool▁call▁begin｜>{} are markup.\"}{} are markup.\"}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v31-truncated-mid-arguments", "family": "deepseek", "status": "fail", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists." }, { "check": "expected_error", "strategy": "special", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists." }, { "check": "expected_error", "strategy": "token", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists." }, { "check": "expected_error", "strategy": "rand:1:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists." }, { "check": "expected_error", "strategy": "rand:2:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists." }, { "check": "expected_error", "strategy": "rand:3:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists." }, { "check": "expected_error", "strategy": "rand:4:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists." }, { "check": "expected_error", "strategy": "rand:5:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists." }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '{}', got '{\"city\": \"Zür' | rand:1:8 (fail): tool_calls[0].arguments: one '{}', got '{\"city\": \"Zür' | rand:2:8 (fail): tool_calls[0].arguments: one '{}', got '{\"city\": \"Zür' | rand:3:8 (fail): tool_calls[0].arguments: one '{}', got '{\"city\": \"Zür' | rand:4:8 (fail): tool_calls[0].arguments: one '{}', got '{\"city\": \"Zür' | rand:5:8 (fail): tool_calls[0].arguments: one '{}', got '{\"city\": \"Zür'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Zür': Unterminated string starting at: line 1 column 10 (char 9))" }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Zür': Unterminated string starting at: line 1 column 10 (char 9))" }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Zür': Unterminated string starting at: line 1 column 10 (char 9))" }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Zür': Unterminated string starting at: line 1 column 10 (char 9))" }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Zür': Unterminated string starting at: line 1 column 10 (char 9))" }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Zür': Unterminated string starting at: line 1 column 10 (char 9))" }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: $: 'city' is a required property" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: $: 'city' is a required property" }, { "check": "arguments_schema", "strategy": "token", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv31", "reasoning_parser": "deepseek-v3", "tool_call_detector": "DeepSeekV31Detector", "reasoning_detector": "_DeepSeekV3Detector", "auto_detected": { "tool_call_parser": "deepseekv31", "reasoning_parser": "deepseek-v3" }, "model": "deepseek-ai/DeepSeek-V3.1", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3.1", "revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "deepseek_v3" }, "hf_config_error": null, "chat_template_sha256": "45690185067ea25c073bff2d3e425a45ee2564d2bbf6cdf7d885e139d460b8e0", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128799 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zür" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zür" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zür" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zür" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zür" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zür" } ], "exception": null } } } }, { "fixture_id": "deepseek/v31-multi-turn-after-tool-result", "family": "deepseek", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"query\": \"café Zürich\"}', got '{}'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"query\": \"café Zürich\"}', got '{}'" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"query\": \"café Zürich\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"query\": \"café Zürich\"}', got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '{}', got '{\"query\": \"café Zürich\"}' | rand:1:8 (fail): tool_calls[0].arguments: one '{}', got '{\"query\": \"café Zürich\"}' | rand:2:8 (fail): tool_calls[0].arguments: one '{}', got '{\"query\": \"café Zürich\"}' | rand:3:8 (fail): tool_calls[0].arguments: one '{}', got '{\"query\": \"café Zürich\"}' | rand:4:8 (fail): tool_calls[0].arguments: one '{}', got '{\"query\": \"café Zürich\"}' | rand:5:8 (fail): tool_calls[0].arguments: one '{}', got '{\"query\": \"café Zürich\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] search: $: 'query' is a required property" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] search: $: 'query' is a required property" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv31", "reasoning_parser": "deepseek-v3", "tool_call_detector": "DeepSeekV31Detector", "reasoning_detector": "_DeepSeekV3Detector", "auto_detected": { "tool_call_parser": "deepseekv31", "reasoning_parser": "deepseek-v3" }, "model": "deepseek-ai/DeepSeek-V3.1", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3.1", "revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "deepseek_v3" }, "hf_config_error": null, "chat_template_sha256": "45690185067ea25c073bff2d3e425a45ee2564d2bbf6cdf7d885e139d460b8e0", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "chat_template", "ids": [ 26606, 91294, 11, 128804, 128799 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"café Zürich\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "search", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "search", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"café Zürich\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"café Zürich\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"café Zürich\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"café Zürich\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"café Zürich\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"café Zürich\"}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v31-nested-json", "family": "deepseek", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam..., got '{}'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam..., got '{}'" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam..., got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam..., got '{}'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '{}', got '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam... | rand:1:8 (fail): tool_calls[0].arguments: one '{}', got '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam... | rand:2:8 (fail): tool_calls[0].arguments: one '{}', got '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam... | rand:3:8 (fail): tool_calls[0].arguments: one '{}', got '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam... | rand:4:8 (fail): tool_calls[0].arguments: one '{}', got '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam... | rand:5:8 (fail): tool_calls[0].arguments: one '{}', got '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"nam..." }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] create_event: $: 'title' is a required property; $: 'when' is a required property" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] create_event: $: 'title' is a required property; $: 'when' is a required property" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv31", "reasoning_parser": "deepseek-v3", "tool_call_detector": "DeepSeekV31Detector", "reasoning_detector": "_DeepSeekV3Detector", "auto_detected": { "tool_call_parser": "deepseekv31", "reasoning_parser": "deepseek-v3" }, "model": "deepseek-ai/DeepSeek-V3.1", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3.1", "revision": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "deepseek_v3" }, "hf_config_error": null, "chat_template_sha256": "45690185067ea25c073bff2d3e425a45ee2564d2bbf6cdf7d885e139d460b8e0", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128799 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v32-single-call", "family": "deepseek", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one None, got '\\n\\n\\n' | rand:2:8 (soft_pass): content (whitespace only): one None, got '\\n\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv32", "reasoning_parser": "deepseek-v3", "tool_call_detector": "DeepSeekV32Detector", "reasoning_detector": "_DeepSeekV3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3.2", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3.2", "revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV32ForCausalLM" ], "model_type": "deepseek_v32" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv32", "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128799 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "token": { "content": "\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v32-parallel-two-calls", "family": "deepseek", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one None, got '\\n\\n\\n\\n' | rand:1:8 (soft_pass): content (whitespace only): one None, got '\\n' | rand:2:8 (soft_pass): content (whitespace only): one None, got '\\n\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv32", "reasoning_parser": "deepseek-v3", "tool_call_detector": "DeepSeekV32Detector", "reasoning_detector": "_DeepSeekV3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3.2", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3.2", "revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV32ForCausalLM" ], "model_type": "deepseek_v32" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv32", "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128799 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "token": { "content": "\n\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:1:8": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v32-no-call", "family": "deepseek", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv32", "reasoning_parser": "deepseek-v3", "tool_call_detector": "DeepSeekV32Detector", "reasoning_detector": "_DeepSeekV3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3.2", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3.2", "revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV32ForCausalLM" ], "model_type": "deepseek_v32" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv32", "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128799 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "deepseek/v32-text-before-call", "family": "deepseek", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n\\n' | rand:4:8 (soft_pass): content (whitespace only): one 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv32", "reasoning_parser": "deepseek-v3", "tool_call_detector": "DeepSeekV32Detector", "reasoning_detector": "_DeepSeekV3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3.2", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3.2", "revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV32ForCausalLM" ], "model_type": "deepseek_v32" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv32", "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128799 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "Let me check the current conditions in London.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "streams": { "one": { "content": "Let me check the current conditions in London.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "special": { "content": "Let me check the current conditions in London.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "token": { "content": "Let me check the current conditions in London.\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:1:8": { "content": "Let me check the current conditions in London.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:2:8": { "content": "Let me check the current conditions in London.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:3:8": { "content": "Let me check the current conditions in London.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:4:8": { "content": "Let me check the current conditions in London.\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:5:8": { "content": "Let me check the current conditions in London.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v32-empty-arguments", "family": "deepseek", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one None, got '\\n\\n\\n' | rand:2:8 (soft_pass): content (whitespace only): one None, got '\\n\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv32", "reasoning_parser": "deepseek-v3", "tool_call_detector": "DeepSeekV32Detector", "reasoning_detector": "_DeepSeekV3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3.2", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3.2", "revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV32ForCausalLM" ], "model_type": "deepseek_v32" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv32", "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128799 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": "\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v32-numeric-arguments", "family": "deepseek", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one None, got '\\n\\n\\n' | rand:2:8 (soft_pass): content (whitespace only): one None, got '\\n\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv32", "reasoning_parser": "deepseek-v3", "tool_call_detector": "DeepSeekV32Detector", "reasoning_detector": "_DeepSeekV3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3.2", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3.2", "revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV32ForCausalLM" ], "model_type": "deepseek_v32" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv32", "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128799 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}" } ], "exception": null }, "token": { "content": "\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v32-unicode-emoji", "family": "deepseek", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one None, got '\\n\\n\\n' | rand:2:8 (soft_pass): content (whitespace only): one None, got '\\n\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv32", "reasoning_parser": "deepseek-v3", "tool_call_detector": "DeepSeekV32Detector", "reasoning_detector": "_DeepSeekV3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3.2", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3.2", "revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV32ForCausalLM" ], "model_type": "deepseek_v32" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv32", "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128799 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null }, "token": { "content": "\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v32-marker-in-arguments", "family": "deepseek", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one None, got '\\n\\n\\n' | rand:2:8 (soft_pass): content (whitespace only): one None, got '\\n\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv32", "reasoning_parser": "deepseek-v3", "tool_call_detector": "DeepSeekV32Detector", "reasoning_detector": "_DeepSeekV3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3.2", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3.2", "revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV32ForCausalLM" ], "model_type": "deepseek_v32" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv32", "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128799 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Write <｜DSML｜invoke name=\\\"x\\\"> to call a tool.\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Write <｜DSML｜invoke name=\\\"x\\\"> to call a tool.\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Write <｜DSML｜invoke name=\\\"x\\\"> to call a tool.\"}" } ], "exception": null }, "token": { "content": "\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Write <｜DSML｜invoke name=\\\"x\\\"> to call a tool.\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Write <｜DSML｜invoke name=\\\"x\\\"> to call a tool.\"}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Write <｜DSML｜invoke name=\\\"x\\\"> to call a tool.\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Write <｜DSML｜invoke name=\\\"x\\\"> to call a tool.\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Write <｜DSML｜invoke name=\\\"x\\\"> to call a tool.\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Write <｜DSML｜invoke name=\\\"x\\\"> to call a tool.\"}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v32-truncated-mid-arguments", "family": "deepseek", "status": "fail", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists." }, { "check": "expected_error", "strategy": "special", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists." }, { "check": "expected_error", "strategy": "token", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists." }, { "check": "expected_error", "strategy": "rand:1:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists." }, { "check": "expected_error", "strategy": "rand:2:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists." }, { "check": "expected_error", "strategy": "rand:3:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists." }, { "check": "expected_error", "strategy": "rand:4:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists." }, { "check": "expected_error", "strategy": "rand:5:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists." }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content (whitespace only): nonstream None, got '\\n\\n'; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content (whitespace only): nonstream None, got '\\n\\n'; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one None, got '\\n\\n' | rand:2:8 (soft_pass): content (whitespace only): one None, got '\\n\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv32", "reasoning_parser": "deepseek-v3", "tool_call_detector": "DeepSeekV32Detector", "reasoning_detector": "_DeepSeekV3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3.2", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3.2", "revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV32ForCausalLM" ], "model_type": "deepseek_v32" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv32", "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128799 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zü\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zü\"}" } ], "exception": null }, "token": { "content": "\n\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zü\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zü\"}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zü\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zü\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zü\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zü\"}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v32-reasoning-single-call", "family": "deepseek", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one None, got '\\n\\n\\n' | rand:4:8 (soft_pass): content (whitespace only): one None, got '\\n\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv32", "reasoning_parser": "deepseek-v3", "tool_call_detector": "DeepSeekV32Detector", "reasoning_detector": "_DeepSeekV3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3.2", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3.2", "revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV32ForCausalLM" ], "model_type": "deepseek_v32" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv32", "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128798 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "token": { "content": "\n\n\n", "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:4:8": { "content": "\n\n", "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v32-reasoning-parallel-text-before", "family": "deepseek", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected \"I'll look both up.\", got \"I'll look both up.\\n\\n\\n\\n\"" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream \"I'll look both up.\", got \"I'll look both up.\\n\\n\\n\\n\"" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one \"I'll look both up.\", got \"I'll look both up.\\n\\n\\n\\n\"" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv32", "reasoning_parser": "deepseek-v3", "tool_call_detector": "DeepSeekV32Detector", "reasoning_detector": "_DeepSeekV3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3.2", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3.2", "revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV32ForCausalLM" ], "model_type": "deepseek_v32" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv32", "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128798 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "I'll look both up.", "reasoning_content": "Two independent lookups: weather and search.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "streams": { "one": { "content": "I'll look both up.", "reasoning_content": "Two independent lookups: weather and search.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "special": { "content": "I'll look both up.", "reasoning_content": "Two independent lookups: weather and search.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "token": { "content": "I'll look both up.\n\n\n\n", "reasoning_content": "Two independent lookups: weather and search.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:1:8": { "content": "I'll look both up.", "reasoning_content": "Two independent lookups: weather and search.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:2:8": { "content": "I'll look both up.", "reasoning_content": "Two independent lookups: weather and search.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:3:8": { "content": "I'll look both up.", "reasoning_content": "Two independent lookups: weather and search.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:4:8": { "content": "I'll look both up.", "reasoning_content": "Two independent lookups: weather and search.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:5:8": { "content": "I'll look both up.", "reasoning_content": "Two independent lookups: weather and search.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v32-reasoning-no-call", "family": "deepseek", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv32", "reasoning_parser": "deepseek-v3", "tool_call_detector": "DeepSeekV32Detector", "reasoning_detector": "_DeepSeekV3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3.2", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3.2", "revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV32ForCausalLM" ], "model_type": "deepseek_v32" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv32", "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128798 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "deepseek/v32-string-false-json", "family": "deepseek", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one None, got '\\n\\n\\n' | rand:2:8 (soft_pass): content (whitespace only): one None, got '\\n\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv32", "reasoning_parser": "deepseek-v3", "tool_call_detector": "DeepSeekV32Detector", "reasoning_detector": "_DeepSeekV3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3.2", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3.2", "revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV32ForCausalLM" ], "model_type": "deepseek_v32" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv32", "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128799 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}" } ], "exception": null }, "token": { "content": "\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v32-unescaped-string-value", "family": "deepseek", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\\\\n\"}', got '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\"}'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\\\\n\"}', got '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\"}'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\\\\n\"}', got '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\"}'" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'; tool_calls[0].arguments: expected '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\\\\n\"}', got '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\"}'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\\\\n\"}', got '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\"}'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content (whitespace only): expected None, got '\\n\\n'; tool_calls[0].arguments: expected '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\\\\n\"}', got '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\"}'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\\\\n\"}', got '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\"}'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\\\\n\"}', got '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\"}'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\\\\n\"}', got '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\"}'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one None, got '\\n\\n\\n' | rand:2:8 (soft_pass): content (whitespace only): one None, got '\\n\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv32", "reasoning_parser": "deepseek-v3", "tool_call_detector": "DeepSeekV32Detector", "reasoning_detector": "_DeepSeekV3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3.2", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3.2", "revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV32ForCausalLM" ], "model_type": "deepseek_v32" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv32", "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128799 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\n print(\\\"x=\\\\\\\"1\\\\\\\"\\\") # tab\\there\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\n print(\\\"x=\\\\\\\"1\\\\\\\"\\\") # tab\\there\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\n print(\\\"x=\\\\\\\"1\\\\\\\"\\\") # tab\\there\"}" } ], "exception": null }, "token": { "content": "\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\n print(\\\"x=\\\\\\\"1\\\\\\\"\\\") # tab\\there\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\n print(\\\"x=\\\\\\\"1\\\\\\\"\\\") # tab\\there\"}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\n print(\\\"x=\\\\\\\"1\\\\\\\"\\\") # tab\\there\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\n print(\\\"x=\\\\\\\"1\\\\\\\"\\\") # tab\\there\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\n print(\\\"x=\\\\\\\"1\\\\\\\"\\\") # tab\\there\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\n print(\\\"x=\\\\\\\"1\\\\\\\"\\\") # tab\\there\"}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v32-truncated-in-reasoning", "family": "deepseek", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv32", "reasoning_parser": "deepseek-v3", "tool_call_detector": "DeepSeekV32Detector", "reasoning_detector": "_DeepSeekV3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3.2", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3.2", "revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV32ForCausalLM" ], "model_type": "deepseek_v32" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv32", "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128798 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "deepseek/v32-truncated-second-parallel-call", "family": "deepseek", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'; tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content (whitespace only): expected None, got '\\n\\n'; tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather', 'search']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather', 'search']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content (whitespace only): nonstream None, got '\\n\\n\\n'; tool_calls: nonstream [], got ['get_weather', 'search']" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather', 'search']" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content (whitespace only): nonstream None, got '\\n\\n'; tool_calls: nonstream [], got ['get_weather', 'search']" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather', 'search']" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather', 'search']" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather', 'search']" }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one None, got '\\n\\n\\n' | rand:2:8 (soft_pass): content (whitespace only): one None, got '\\n\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[1] search: $: 'query' is a required property" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[1] search: $: 'query' is a required property" }, { "check": "arguments_schema", "strategy": "token", "status": "fail", "detail": "[1] search: $: 'query' is a required property" }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "fail", "detail": "[1] search: $: 'query' is a required property" }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "fail", "detail": "[1] search: $: 'query' is a required property" }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "fail", "detail": "[1] search: $: 'query' is a required property" }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[1] search: $: 'query' is a required property" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "fail", "detail": "[1] search: $: 'query' is a required property" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv32", "reasoning_parser": "deepseek-v3", "tool_call_detector": "DeepSeekV32Detector", "reasoning_detector": "_DeepSeekV3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V3.2", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V3.2", "revision": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV32ForCausalLM" ], "model_type": "deepseek_v32" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv32", "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128799 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": "\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v4-single-call", "family": "deepseek", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one None, got '\\n\\n\\n' | rand:2:8 (soft_pass): content (whitespace only): one None, got '\\n\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv4", "reasoning_parser": "deepseek-v4", "tool_call_detector": "DeepSeekV4Detector", "reasoning_detector": "DeepSeekV4Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V4-Flash", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V4-Flash", "revision": "60d8d70770c6776ff598c94bb586a859a38244f1", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV4ForCausalLM" ], "model_type": "deepseek_v4" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv4", "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128822 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "token": { "content": "\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v4-parallel-two-calls", "family": "deepseek", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one None, got '\\n\\n\\n\\n' | rand:1:8 (soft_pass): content (whitespace only): one None, got '\\n' | rand:2:8 (soft_pass): content (whitespace only): one None, got '\\n\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv4", "reasoning_parser": "deepseek-v4", "tool_call_detector": "DeepSeekV4Detector", "reasoning_detector": "DeepSeekV4Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V4-Flash", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V4-Flash", "revision": "60d8d70770c6776ff598c94bb586a859a38244f1", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV4ForCausalLM" ], "model_type": "deepseek_v4" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv4", "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128822 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "token": { "content": "\n\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:1:8": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v4-no-call", "family": "deepseek", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv4", "reasoning_parser": "deepseek-v4", "tool_call_detector": "DeepSeekV4Detector", "reasoning_detector": "DeepSeekV4Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V4-Flash", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V4-Flash", "revision": "60d8d70770c6776ff598c94bb586a859a38244f1", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV4ForCausalLM" ], "model_type": "deepseek_v4" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv4", "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128822 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "deepseek/v4-text-before-call", "family": "deepseek", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n\\n' | rand:4:8 (soft_pass): content (whitespace only): one 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n\\n\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv4", "reasoning_parser": "deepseek-v4", "tool_call_detector": "DeepSeekV4Detector", "reasoning_detector": "DeepSeekV4Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V4-Flash", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V4-Flash", "revision": "60d8d70770c6776ff598c94bb586a859a38244f1", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV4ForCausalLM" ], "model_type": "deepseek_v4" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv4", "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128822 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "Let me check the current conditions in London.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "streams": { "one": { "content": "Let me check the current conditions in London.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "special": { "content": "Let me check the current conditions in London.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "token": { "content": "Let me check the current conditions in London.\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:1:8": { "content": "Let me check the current conditions in London.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:2:8": { "content": "Let me check the current conditions in London.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:3:8": { "content": "Let me check the current conditions in London.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:4:8": { "content": "Let me check the current conditions in London.\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:5:8": { "content": "Let me check the current conditions in London.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v4-empty-arguments", "family": "deepseek", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one None, got '\\n\\n\\n' | rand:2:8 (soft_pass): content (whitespace only): one None, got '\\n\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv4", "reasoning_parser": "deepseek-v4", "tool_call_detector": "DeepSeekV4Detector", "reasoning_detector": "DeepSeekV4Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V4-Flash", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V4-Flash", "revision": "60d8d70770c6776ff598c94bb586a859a38244f1", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV4ForCausalLM" ], "model_type": "deepseek_v4" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv4", "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128822 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": "\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v4-numeric-arguments", "family": "deepseek", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one None, got '\\n\\n\\n' | rand:2:8 (soft_pass): content (whitespace only): one None, got '\\n\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv4", "reasoning_parser": "deepseek-v4", "tool_call_detector": "DeepSeekV4Detector", "reasoning_detector": "DeepSeekV4Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V4-Flash", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V4-Flash", "revision": "60d8d70770c6776ff598c94bb586a859a38244f1", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV4ForCausalLM" ], "model_type": "deepseek_v4" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv4", "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128822 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}" } ], "exception": null }, "token": { "content": "\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21], \"code\": \"007\"}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v4-unicode-emoji", "family": "deepseek", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one None, got '\\n\\n\\n' | rand:2:8 (soft_pass): content (whitespace only): one None, got '\\n\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv4", "reasoning_parser": "deepseek-v4", "tool_call_detector": "DeepSeekV4Detector", "reasoning_detector": "DeepSeekV4Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V4-Flash", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V4-Flash", "revision": "60d8d70770c6776ff598c94bb586a859a38244f1", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV4ForCausalLM" ], "model_type": "deepseek_v4" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv4", "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128822 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null }, "token": { "content": "\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v4-marker-in-arguments", "family": "deepseek", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one None, got '\\n\\n\\n' | rand:2:8 (soft_pass): content (whitespace only): one None, got '\\n\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv4", "reasoning_parser": "deepseek-v4", "tool_call_detector": "DeepSeekV4Detector", "reasoning_detector": "DeepSeekV4Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V4-Flash", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V4-Flash", "revision": "60d8d70770c6776ff598c94bb586a859a38244f1", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV4ForCausalLM" ], "model_type": "deepseek_v4" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv4", "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128822 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Write <｜DSML｜invoke name=\\\"x\\\"> inside <｜DSML｜tool_calls> to call a tool.\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Write <｜DSML｜invoke name=\\\"x\\\"> inside <｜DSML｜tool_calls> to call a tool.\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Write <｜DSML｜invoke name=\\\"x\\\"> inside <｜DSML｜tool_calls> to call a tool.\"}" } ], "exception": null }, "token": { "content": "\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Write <｜DSML｜invoke name=\\\"x\\\"> inside <｜DSML｜tool_calls> to call a tool.\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Write <｜DSML｜invoke name=\\\"x\\\"> inside <｜DSML｜tool_calls> to call a tool.\"}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Write <｜DSML｜invoke name=\\\"x\\\"> inside <｜DSML｜tool_calls> to call a tool.\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Write <｜DSML｜invoke name=\\\"x\\\"> inside <｜DSML｜tool_calls> to call a tool.\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Write <｜DSML｜invoke name=\\\"x\\\"> inside <｜DSML｜tool_calls> to call a tool.\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Write <｜DSML｜invoke name=\\\"x\\\"> inside <｜DSML｜tool_calls> to call a tool.\"}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v4-truncated-mid-arguments", "family": "deepseek", "status": "fail", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists." }, { "check": "expected_error", "strategy": "special", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists." }, { "check": "expected_error", "strategy": "token", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists." }, { "check": "expected_error", "strategy": "rand:1:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists." }, { "check": "expected_error", "strategy": "rand:2:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists." }, { "check": "expected_error", "strategy": "rand:3:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists." }, { "check": "expected_error", "strategy": "rand:4:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists." }, { "check": "expected_error", "strategy": "rand:5:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the first call's arguments: no complete call exists." }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content (whitespace only): nonstream None, got '\\n\\n'; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content (whitespace only): nonstream None, got '\\n\\n'; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one None, got '\\n\\n' | rand:2:8 (soft_pass): content (whitespace only): one None, got '\\n\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv4", "reasoning_parser": "deepseek-v4", "tool_call_detector": "DeepSeekV4Detector", "reasoning_detector": "DeepSeekV4Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V4-Flash", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V4-Flash", "revision": "60d8d70770c6776ff598c94bb586a859a38244f1", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV4ForCausalLM" ], "model_type": "deepseek_v4" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv4", "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128822 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zü\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zü\"}" } ], "exception": null }, "token": { "content": "\n\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zü\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zü\"}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zü\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zü\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zü\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zü\"}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v4-multi-turn-after-tool-result", "family": "deepseek", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one None, got '\\n\\n\\n' | rand:3:8 (soft_pass): content (whitespace only): one None, got '\\n\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv4", "reasoning_parser": "deepseek-v4", "tool_call_detector": "DeepSeekV4Detector", "reasoning_detector": "DeepSeekV4Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V4-Flash", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V4-Flash", "revision": "60d8d70770c6776ff598c94bb586a859a38244f1", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV4ForCausalLM" ], "model_type": "deepseek_v4" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv4", "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128821 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "It is 20C. Now the café search.", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"café Zürich\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "It is 20C. Now the café search.", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"café Zürich\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "It is 20C. Now the café search.", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"café Zürich\"}" } ], "exception": null }, "token": { "content": "\n\n\n", "reasoning_content": "It is 20C. Now the café search.", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"café Zürich\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "It is 20C. Now the café search.", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"café Zürich\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "It is 20C. Now the café search.", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"café Zürich\"}" } ], "exception": null }, "rand:3:8": { "content": "\n\n", "reasoning_content": "It is 20C. Now the café search.", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"café Zürich\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "It is 20C. Now the café search.", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"café Zürich\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "It is 20C. Now the café search.", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"café Zürich\"}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v4-reasoning-single-call", "family": "deepseek", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one None, got '\\n\\n\\n' | rand:4:8 (soft_pass): content (whitespace only): one None, got '\\n\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv4", "reasoning_parser": "deepseek-v4", "tool_call_detector": "DeepSeekV4Detector", "reasoning_detector": "DeepSeekV4Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V4-Flash", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V4-Flash", "revision": "60d8d70770c6776ff598c94bb586a859a38244f1", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV4ForCausalLM" ], "model_type": "deepseek_v4" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv4", "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128821 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "token": { "content": "\n\n\n", "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:4:8": { "content": "\n\n", "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v4-reasoning-parallel-text-before", "family": "deepseek", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected \"I'll look both up.\", got \"I'll look both up.\\n\\n\\n\\n\"" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream \"I'll look both up.\", got \"I'll look both up.\\n\\n\\n\\n\"" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one \"I'll look both up.\", got \"I'll look both up.\\n\\n\\n\\n\"" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv4", "reasoning_parser": "deepseek-v4", "tool_call_detector": "DeepSeekV4Detector", "reasoning_detector": "DeepSeekV4Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V4-Flash", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V4-Flash", "revision": "60d8d70770c6776ff598c94bb586a859a38244f1", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV4ForCausalLM" ], "model_type": "deepseek_v4" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv4", "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128821 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "I'll look both up.", "reasoning_content": "Two independent lookups: weather and search.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "streams": { "one": { "content": "I'll look both up.", "reasoning_content": "Two independent lookups: weather and search.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "special": { "content": "I'll look both up.", "reasoning_content": "Two independent lookups: weather and search.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "token": { "content": "I'll look both up.\n\n\n\n", "reasoning_content": "Two independent lookups: weather and search.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:1:8": { "content": "I'll look both up.", "reasoning_content": "Two independent lookups: weather and search.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:2:8": { "content": "I'll look both up.", "reasoning_content": "Two independent lookups: weather and search.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:3:8": { "content": "I'll look both up.", "reasoning_content": "Two independent lookups: weather and search.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:4:8": { "content": "I'll look both up.", "reasoning_content": "Two independent lookups: weather and search.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:5:8": { "content": "I'll look both up.", "reasoning_content": "Two independent lookups: weather and search.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v4-reasoning-no-call", "family": "deepseek", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv4", "reasoning_parser": "deepseek-v4", "tool_call_detector": "DeepSeekV4Detector", "reasoning_detector": "DeepSeekV4Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V4-Flash", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V4-Flash", "revision": "60d8d70770c6776ff598c94bb586a859a38244f1", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV4ForCausalLM" ], "model_type": "deepseek_v4" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv4", "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128821 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "deepseek/v4-string-false-json", "family": "deepseek", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one None, got '\\n\\n\\n' | rand:2:8 (soft_pass): content (whitespace only): one None, got '\\n\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv4", "reasoning_parser": "deepseek-v4", "tool_call_detector": "DeepSeekV4Detector", "reasoning_detector": "DeepSeekV4Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V4-Flash", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V4-Flash", "revision": "60d8d70770c6776ff598c94bb586a859a38244f1", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV4ForCausalLM" ], "model_type": "deepseek_v4" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv4", "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128822 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}" } ], "exception": null }, "token": { "content": "\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\"}, \"attendees\": [{\"name\": \"Ana\", \"optional\": false}, {\"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"]}]}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v4-unescaped-string-value", "family": "deepseek", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\\\\n\"}', got '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\"}'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\\\\n\"}', got '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\"}'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\\\\n\"}', got '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\"}'" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'; tool_calls[0].arguments: expected '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\\\\n\"}', got '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\"}'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\\\\n\"}', got '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\"}'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content (whitespace only): expected None, got '\\n\\n'; tool_calls[0].arguments: expected '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\\\\n\"}', got '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\"}'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\\\\n\"}', got '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\"}'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\\\\n\"}', got '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\"}'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\\\\n\"}', got '{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\\\n print(\\\\\"x=\\\\\\\\\\\\\"1\\\\\\\\\\\\\"\\\\\") # tab\\\\there\"}'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one None, got '\\n\\n\\n' | rand:2:8 (soft_pass): content (whitespace only): one None, got '\\n\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv4", "reasoning_parser": "deepseek-v4", "tool_call_detector": "DeepSeekV4Detector", "reasoning_detector": "DeepSeekV4Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V4-Flash", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V4-Flash", "revision": "60d8d70770c6776ff598c94bb586a859a38244f1", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV4ForCausalLM" ], "model_type": "deepseek_v4" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv4", "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128822 ] }, "stop": { "token": "<｜end▁of▁sentence｜>", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\n print(\\\"x=\\\\\\\"1\\\\\\\"\\\") # tab\\there\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\n print(\\\"x=\\\\\\\"1\\\\\\\"\\\") # tab\\there\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\n print(\\\"x=\\\\\\\"1\\\\\\\"\\\") # tab\\there\"}" } ], "exception": null }, "token": { "content": "\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\n print(\\\"x=\\\\\\\"1\\\\\\\"\\\") # tab\\there\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\n print(\\\"x=\\\\\\\"1\\\\\\\"\\\") # tab\\there\"}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\n print(\\\"x=\\\\\\\"1\\\\\\\"\\\") # tab\\there\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\n print(\\\"x=\\\\\\\"1\\\\\\\"\\\") # tab\\there\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\n print(\\\"x=\\\\\\\"1\\\\\\\"\\\") # tab\\there\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"a.py\", \"content\": \"if a < b and c > d:\\n print(\\\"x=\\\\\\\"1\\\\\\\"\\\") # tab\\there\"}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v4-truncated-in-reasoning", "family": "deepseek", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv4", "reasoning_parser": "deepseek-v4", "tool_call_detector": "DeepSeekV4Detector", "reasoning_detector": "DeepSeekV4Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V4-Flash", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V4-Flash", "revision": "60d8d70770c6776ff598c94bb586a859a38244f1", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV4ForCausalLM" ], "model_type": "deepseek_v4" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv4", "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128821 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "deepseek/v4-truncated-second-parallel-call", "family": "deepseek", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'; tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content (whitespace only): expected None, got '\\n\\n'; tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather', 'search']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather', 'search']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content (whitespace only): nonstream None, got '\\n\\n\\n'; tool_calls: nonstream [], got ['get_weather', 'search']" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather', 'search']" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content (whitespace only): nonstream None, got '\\n\\n'; tool_calls: nonstream [], got ['get_weather', 'search']" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather', 'search']" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather', 'search']" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather', 'search']" }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one None, got '\\n\\n\\n' | rand:2:8 (soft_pass): content (whitespace only): one None, got '\\n\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[1] search: $: 'query' is a required property" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[1] search: $: 'query' is a required property" }, { "check": "arguments_schema", "strategy": "token", "status": "fail", "detail": "[1] search: $: 'query' is a required property" }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "fail", "detail": "[1] search: $: 'query' is a required property" }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "fail", "detail": "[1] search: $: 'query' is a required property" }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "fail", "detail": "[1] search: $: 'query' is a required property" }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[1] search: $: 'query' is a required property" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "fail", "detail": "[1] search: $: 'query' is a required property" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "deepseekv4", "reasoning_parser": "deepseek-v4", "tool_call_detector": "DeepSeekV4Detector", "reasoning_detector": "DeepSeekV4Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "deepseek-ai/DeepSeek-V4-Flash", "tokenizer": { "repo": "deepseek-ai/DeepSeek-V4-Flash", "revision": "60d8d70770c6776ff598c94bb586a859a38244f1", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV4ForCausalLM" ], "model_type": "deepseek_v4" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "dsv4", "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128804, 128822 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": "\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{}" } ], "exception": null } } } }, { "fixture_id": "deepseek/v41-single-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "SGLang 0.5.20 has no DeepSeek-V4.1 parser: V4.1's spaced DSML markers ('DSML' + U+FF5C + ' calls') appear nowhere in sglang/srt; the deepseekv4 detector only knows V4's 'DSML' + U+FF5C + 'tool_calls'" }, { "fixture_id": "deepseek/v41-parallel-two-calls", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "SGLang 0.5.20 has no DeepSeek-V4.1 parser: V4.1's spaced DSML markers ('DSML' + U+FF5C + ' calls') appear nowhere in sglang/srt; the deepseekv4 detector only knows V4's 'DSML' + U+FF5C + 'tool_calls'" }, { "fixture_id": "deepseek/v41-no-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "SGLang 0.5.20 has no DeepSeek-V4.1 parser: V4.1's spaced DSML markers ('DSML' + U+FF5C + ' calls') appear nowhere in sglang/srt; the deepseekv4 detector only knows V4's 'DSML' + U+FF5C + 'tool_calls'" }, { "fixture_id": "deepseek/v41-text-before-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "SGLang 0.5.20 has no DeepSeek-V4.1 parser: V4.1's spaced DSML markers ('DSML' + U+FF5C + ' calls') appear nowhere in sglang/srt; the deepseekv4 detector only knows V4's 'DSML' + U+FF5C + 'tool_calls'" }, { "fixture_id": "deepseek/v41-empty-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "SGLang 0.5.20 has no DeepSeek-V4.1 parser: V4.1's spaced DSML markers ('DSML' + U+FF5C + ' calls') appear nowhere in sglang/srt; the deepseekv4 detector only knows V4's 'DSML' + U+FF5C + 'tool_calls'" }, { "fixture_id": "deepseek/v41-numeric-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "SGLang 0.5.20 has no DeepSeek-V4.1 parser: V4.1's spaced DSML markers ('DSML' + U+FF5C + ' calls') appear nowhere in sglang/srt; the deepseekv4 detector only knows V4's 'DSML' + U+FF5C + 'tool_calls'" }, { "fixture_id": "deepseek/v41-unicode-emoji", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "SGLang 0.5.20 has no DeepSeek-V4.1 parser: V4.1's spaced DSML markers ('DSML' + U+FF5C + ' calls') appear nowhere in sglang/srt; the deepseekv4 detector only knows V4's 'DSML' + U+FF5C + 'tool_calls'" }, { "fixture_id": "deepseek/v41-marker-in-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "SGLang 0.5.20 has no DeepSeek-V4.1 parser: V4.1's spaced DSML markers ('DSML' + U+FF5C + ' calls') appear nowhere in sglang/srt; the deepseekv4 detector only knows V4's 'DSML' + U+FF5C + 'tool_calls'" }, { "fixture_id": "deepseek/v41-truncated-mid-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "SGLang 0.5.20 has no DeepSeek-V4.1 parser: V4.1's spaced DSML markers ('DSML' + U+FF5C + ' calls') appear nowhere in sglang/srt; the deepseekv4 detector only knows V4's 'DSML' + U+FF5C + 'tool_calls'" }, { "fixture_id": "deepseek/v41-multi-turn-after-tool-result", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "SGLang 0.5.20 has no DeepSeek-V4.1 parser: V4.1's spaced DSML markers ('DSML' + U+FF5C + ' calls') appear nowhere in sglang/srt; the deepseekv4 detector only knows V4's 'DSML' + U+FF5C + 'tool_calls'" }, { "fixture_id": "deepseek/v41-reasoning-single-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "SGLang 0.5.20 has no DeepSeek-V4.1 parser: V4.1's spaced DSML markers ('DSML' + U+FF5C + ' calls') appear nowhere in sglang/srt; the deepseekv4 detector only knows V4's 'DSML' + U+FF5C + 'tool_calls'" }, { "fixture_id": "deepseek/v41-reasoning-parallel-text-before", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "SGLang 0.5.20 has no DeepSeek-V4.1 parser: V4.1's spaced DSML markers ('DSML' + U+FF5C + ' calls') appear nowhere in sglang/srt; the deepseekv4 detector only knows V4's 'DSML' + U+FF5C + 'tool_calls'" }, { "fixture_id": "deepseek/v41-reasoning-no-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "SGLang 0.5.20 has no DeepSeek-V4.1 parser: V4.1's spaced DSML markers ('DSML' + U+FF5C + ' calls') appear nowhere in sglang/srt; the deepseekv4 detector only knows V4's 'DSML' + U+FF5C + 'tool_calls'" }, { "fixture_id": "deepseek/v41-string-false-json", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "SGLang 0.5.20 has no DeepSeek-V4.1 parser: V4.1's spaced DSML markers ('DSML' + U+FF5C + ' calls') appear nowhere in sglang/srt; the deepseekv4 detector only knows V4's 'DSML' + U+FF5C + 'tool_calls'" }, { "fixture_id": "deepseek/v41-unescaped-string-value", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "SGLang 0.5.20 has no DeepSeek-V4.1 parser: V4.1's spaced DSML markers ('DSML' + U+FF5C + ' calls') appear nowhere in sglang/srt; the deepseekv4 detector only knows V4's 'DSML' + U+FF5C + 'tool_calls'" }, { "fixture_id": "deepseek/v41-truncated-in-reasoning", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "SGLang 0.5.20 has no DeepSeek-V4.1 parser: V4.1's spaced DSML markers ('DSML' + U+FF5C + ' calls') appear nowhere in sglang/srt; the deepseekv4 detector only knows V4's 'DSML' + U+FF5C + 'tool_calls'" }, { "fixture_id": "deepseek/v41-truncated-second-parallel-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "SGLang 0.5.20 has no DeepSeek-V4.1 parser: V4.1's spaced DSML markers ('DSML' + U+FF5C + ' calls') appear nowhere in sglang/srt; the deepseekv4 detector only knows V4's 'DSML' + U+FF5C + 'tool_calls'" }, { "fixture_id": "deepseek/v41-long-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "SGLang 0.5.20 has no DeepSeek-V4.1 parser: V4.1's spaced DSML markers ('DSML' + U+FF5C + ' calls') appear nowhere in sglang/srt; the deepseekv4 detector only knows V4's 'DSML' + U+FF5C + 'tool_calls'" }, { "fixture_id": "deepseek/v41-whitespace-string-value", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "SGLang 0.5.20 has no DeepSeek-V4.1 parser: V4.1's spaced DSML markers ('DSML' + U+FF5C + ' calls') appear nowhere in sglang/srt; the deepseekv4 detector only knows V4's 'DSML' + U+FF5C + 'tool_calls'" }, { "fixture_id": "gemma4/single-call", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gemma4/single-call-thinking", "family": "gemma4", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 105, 4368, 107 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "token": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null } } } }, { "fixture_id": "gemma4/no-call", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gemma4/no-call-thinking", "family": "gemma4", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got 'The user is just greeting me. No tool is needed.\\n'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got 'The user is just greeting me. No tool is needed.\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got 'The user is just greeting me. No tool is needed.\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got 'The user is just greeting me. No tool is needed.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got 'The user is just greeting me. No tool is needed.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got 'The user is just greeting me. No tool is needed.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got 'The user is just greeting me. No tool is needed.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got 'The user is just greeting me. No tool is needed.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got 'The user is just greeting me. No tool is needed.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 105, 4368, 107 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "Hello! How can I help you today?", "reasoning_content": "The user is just greeting me. No tool is needed.\n", "tool_calls": [], "exception": null }, "streams": { "one": { "content": "Hello! How can I help you today?", "reasoning_content": "The user is just greeting me. No tool is needed.\n", "tool_calls": [], "exception": null }, "special": { "content": "Hello! How can I help you today?", "reasoning_content": "The user is just greeting me. No tool is needed.\n", "tool_calls": [], "exception": null }, "token": { "content": "Hello! How can I help you today?", "reasoning_content": "The user is just greeting me. No tool is needed.\n", "tool_calls": [], "exception": null }, "rand:1:8": { "content": "Hello! How can I help you today?", "reasoning_content": "The user is just greeting me. No tool is needed.\n", "tool_calls": [], "exception": null }, "rand:2:8": { "content": "Hello! How can I help you today?", "reasoning_content": "The user is just greeting me. No tool is needed.\n", "tool_calls": [], "exception": null }, "rand:3:8": { "content": "Hello! How can I help you today?", "reasoning_content": "The user is just greeting me. No tool is needed.\n", "tool_calls": [], "exception": null }, "rand:4:8": { "content": "Hello! How can I help you today?", "reasoning_content": "The user is just greeting me. No tool is needed.\n", "tool_calls": [], "exception": null }, "rand:5:8": { "content": "Hello! How can I help you today?", "reasoning_content": "The user is just greeting me. No tool is needed.\n", "tool_calls": [], "exception": null } } } }, { "fixture_id": "gemma4/text-after-call", "family": "gemma4", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got None" }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream None, got 'Let me check the current conditions in London.'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream None, got 'Let me check the current conditions in London.'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream None, got 'Let me check the current conditions in London.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream None, got 'Let me check the current conditions in London.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream None, got 'Let me check the current conditions in London.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream None, got 'Let me check the current conditions in London.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream None, got 'Let me check the current conditions in London.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream None, got 'Let me check the current conditions in London.'" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "streams": { "one": { "content": "Let me check the current conditions in London.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "special": { "content": "Let me check the current conditions in London.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "token": { "content": "Let me check the current conditions in London.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:1:8": { "content": "Let me check the current conditions in London.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:2:8": { "content": "Let me check the current conditions in London.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:3:8": { "content": "Let me check the current conditions in London.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:4:8": { "content": "Let me check the current conditions in London.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:5:8": { "content": "Let me check the current conditions in London.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null } } } }, { "fixture_id": "gemma4/text-after-call-thinking", "family": "gemma4", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got None; reasoning_content (whitespace only): expected 'I need live weather data for London.', got 'I need live weather data for London.\\n'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I need live weather data for London.', got 'I need live weather data for London.\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I need live weather data for London.', got 'I need live weather data for London.\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I need live weather data for London.', got 'I need live weather data for London.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I need live weather data for London.', got 'I need live weather data for London.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I need live weather data for London.', got 'I need live weather data for London.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I need live weather data for London.', got 'I need live weather data for London.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I need live weather data for London.', got 'I need live weather data for London.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I need live weather data for London.', got 'I need live weather data for London.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream None, got 'Let me check the current conditions in London.'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream None, got 'Let me check the current conditions in London.'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream None, got 'Let me check the current conditions in London.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream None, got 'Let me check the current conditions in London.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream None, got 'Let me check the current conditions in London.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream None, got 'Let me check the current conditions in London.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream None, got 'Let me check the current conditions in London.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream None, got 'Let me check the current conditions in London.'" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 105, 4368, 107 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "I need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "streams": { "one": { "content": "Let me check the current conditions in London.", "reasoning_content": "I need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "special": { "content": "Let me check the current conditions in London.", "reasoning_content": "I need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "token": { "content": "Let me check the current conditions in London.", "reasoning_content": "I need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:1:8": { "content": "Let me check the current conditions in London.", "reasoning_content": "I need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:2:8": { "content": "Let me check the current conditions in London.", "reasoning_content": "I need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:3:8": { "content": "Let me check the current conditions in London.", "reasoning_content": "I need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:4:8": { "content": "Let me check the current conditions in London.", "reasoning_content": "I need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:5:8": { "content": "Let me check the current conditions in London.", "reasoning_content": "I need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null } } } }, { "fixture_id": "gemma4/empty-arguments", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gemma4/empty-arguments-thinking", "family": "gemma4", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'get_time takes no arguments.', got 'get_time takes no arguments.\\n'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'get_time takes no arguments.', got 'get_time takes no arguments.\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'get_time takes no arguments.', got 'get_time takes no arguments.\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'get_time takes no arguments.', got 'get_time takes no arguments.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'get_time takes no arguments.', got 'get_time takes no arguments.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'get_time takes no arguments.', got 'get_time takes no arguments.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'get_time takes no arguments.', got 'get_time takes no arguments.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'get_time takes no arguments.', got 'get_time takes no arguments.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'get_time takes no arguments.', got 'get_time takes no arguments.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 105, 4368, 107 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "get_time takes no arguments.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "get_time takes no arguments.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "get_time takes no arguments.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": null, "reasoning_content": "get_time takes no arguments.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "get_time takes no arguments.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "get_time takes no arguments.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "get_time takes no arguments.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "get_time takes no arguments.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "get_time takes no arguments.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null } } } }, { "fixture_id": "gemma4/bug-ollama-18390-key-with-spaces", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gemma4/nested-objects-and-arrays", "family": "gemma4", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"an..., got '{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": false}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"..." }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"an..., got '{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": false}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"..." }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"an..., got '{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": false}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"..." }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"an..., got '{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": false}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"..." }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"an..., got '{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": false}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"..." }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"an..., got '{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": false}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"..." }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"an..., got '{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": false}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"..." }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"an..., got '{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": false}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"..." }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"an..., got '{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": false}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\"..." }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "fail", "detail": "[0] create_event: $.location: 'null' is not valid under any of the given schemas" }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] create_event: $.location: 'null' is not valid under any of the given schemas" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] create_event: $.location: 'null' is not valid under any of the given schemas" }, { "check": "arguments_schema", "strategy": "token", "status": "fail", "detail": "[0] create_event: $.location: 'null' is not valid under any of the given schemas" }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "fail", "detail": "[0] create_event: $.location: 'null' is not valid under any of the given schemas" }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "fail", "detail": "[0] create_event: $.location: 'null' is not valid under any of the given schemas" }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "fail", "detail": "[0] create_event: $.location: 'null' is not valid under any of the given schemas" }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[0] create_event: $.location: 'null' is not valid under any of the given schemas" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "fail", "detail": "[0] create_event: $.location: 'null' is not valid under any of the given schemas" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": false}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\", \"timekeeper\"]}], \"location\": \"null\", \"reminders\": [[10, \"email\"], [1, \"popup\"]], \"title\": \"Design review\", \"when\": {\"end\": \"2026-10-01T10:00:00Z\", \"start\": \"2026-10-01T09:00:00Z\", \"tz\": \"UTC\"}}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": false}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\", \"timekeeper\"]}], \"location\": \"null\", \"reminders\": [[10, \"email\"], [1, \"popup\"]], \"title\": \"Design review\", \"when\": {\"end\": \"2026-10-01T10:00:00Z\", \"start\": \"2026-10-01T09:00:00Z\", \"tz\": \"UTC\"}}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": false}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\", \"timekeeper\"]}], \"location\": \"null\", \"reminders\": [[10, \"email\"], [1, \"popup\"]], \"title\": \"Design review\", \"when\": {\"end\": \"2026-10-01T10:00:00Z\", \"start\": \"2026-10-01T09:00:00Z\", \"tz\": \"UTC\"}}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": false}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\", \"timekeeper\"]}], \"location\": \"null\", \"reminders\": [[10, \"email\"], [1, \"popup\"]], \"title\": \"Design review\", \"when\": {\"end\": \"2026-10-01T10:00:00Z\", \"start\": \"2026-10-01T09:00:00Z\", \"tz\": \"UTC\"}}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": false}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\", \"timekeeper\"]}], \"location\": \"null\", \"reminders\": [[10, \"email\"], [1, \"popup\"]], \"title\": \"Design review\", \"when\": {\"end\": \"2026-10-01T10:00:00Z\", \"start\": \"2026-10-01T09:00:00Z\", \"tz\": \"UTC\"}}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": false}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\", \"timekeeper\"]}], \"location\": \"null\", \"reminders\": [[10, \"email\"], [1, \"popup\"]], \"title\": \"Design review\", \"when\": {\"end\": \"2026-10-01T10:00:00Z\", \"start\": \"2026-10-01T09:00:00Z\", \"tz\": \"UTC\"}}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": false}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\", \"timekeeper\"]}], \"location\": \"null\", \"reminders\": [[10, \"email\"], [1, \"popup\"]], \"title\": \"Design review\", \"when\": {\"end\": \"2026-10-01T10:00:00Z\", \"start\": \"2026-10-01T09:00:00Z\", \"tz\": \"UTC\"}}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": false}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\", \"timekeeper\"]}], \"location\": \"null\", \"reminders\": [[10, \"email\"], [1, \"popup\"]], \"title\": \"Design review\", \"when\": {\"end\": \"2026-10-01T10:00:00Z\", \"start\": \"2026-10-01T09:00:00Z\", \"tz\": \"UTC\"}}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"attendees\": [{\"email\": \"ana@example.com\", \"name\": \"Ana\", \"optional\": false}, {\"email\": \"bo@example.com\", \"name\": \"Bo\", \"optional\": true, \"roles\": [\"notes\", \"timekeeper\"]}], \"location\": \"null\", \"reminders\": [[10, \"email\"], [1, \"popup\"]], \"title\": \"Design review\", \"when\": {\"end\": \"2026-10-01T10:00:00Z\", \"start\": \"2026-10-01T09:00:00Z\", \"tz\": \"UTC\"}}" } ], "exception": null } } } }, { "fixture_id": "gemma4/edge-values-key-with-space", "family": "gemma4", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"key with space\": 1.5e-05, \"s\": \"x,y} \\\\\"q\\\\\"\", \"flag\": false, \"none\": null, \"obj\": {\"B\": 1, \"a\": [1, \"t\"]}}', got '{\"flag\": false, \"key with space\": 1.5e-05, \"none\": \"null\", \"obj\": {\"a\": [1, \"t\"], \"B\": 1}, \"s\": \"x,y} \\\\\"q\\\\\"\"}'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"key with space\": 1.5e-05, \"s\": \"x,y} \\\\\"q\\\\\"\", \"flag\": false, \"none\": null, \"obj\": {\"B\": 1, \"a\": [1, \"t\"]}}', got '{\"flag\": false, \"key with space\": 1.5e-05, \"none\": \"null\", \"obj\": {\"a\": [1, \"t\"], \"B\": 1}, \"s\": \"x,y} \\\\\"q\\\\\"\"}'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"key with space\": 1.5e-05, \"s\": \"x,y} \\\\\"q\\\\\"\", \"flag\": false, \"none\": null, \"obj\": {\"B\": 1, \"a\": [1, \"t\"]}}', got '{\"flag\": false, \"key with space\": 1.5e-05, \"none\": \"null\", \"obj\": {\"a\": [1, \"t\"], \"B\": 1}, \"s\": \"x,y} \\\\\"q\\\\\"\"}'" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"key with space\": 1.5e-05, \"s\": \"x,y} \\\\\"q\\\\\"\", \"flag\": false, \"none\": null, \"obj\": {\"B\": 1, \"a\": [1, \"t\"]}}', got '{\"flag\": false, \"key with space\": 1.5e-05, \"none\": \"null\", \"obj\": {\"a\": [1, \"t\"], \"B\": 1}, \"s\": \"x,y} \\\\\"q\\\\\"\"}'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"key with space\": 1.5e-05, \"s\": \"x,y} \\\\\"q\\\\\"\", \"flag\": false, \"none\": null, \"obj\": {\"B\": 1, \"a\": [1, \"t\"]}}', got '{\"flag\": false, \"key with space\": 1.5e-05, \"none\": \"null\", \"obj\": {\"a\": [1, \"t\"], \"B\": 1}, \"s\": \"x,y} \\\\\"q\\\\\"\"}'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"key with space\": 1.5e-05, \"s\": \"x,y} \\\\\"q\\\\\"\", \"flag\": false, \"none\": null, \"obj\": {\"B\": 1, \"a\": [1, \"t\"]}}', got '{\"flag\": false, \"key with space\": 1.5e-05, \"none\": \"null\", \"obj\": {\"a\": [1, \"t\"], \"B\": 1}, \"s\": \"x,y} \\\\\"q\\\\\"\"}'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"key with space\": 1.5e-05, \"s\": \"x,y} \\\\\"q\\\\\"\", \"flag\": false, \"none\": null, \"obj\": {\"B\": 1, \"a\": [1, \"t\"]}}', got '{\"flag\": false, \"key with space\": 1.5e-05, \"none\": \"null\", \"obj\": {\"a\": [1, \"t\"], \"B\": 1}, \"s\": \"x,y} \\\\\"q\\\\\"\"}'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"key with space\": 1.5e-05, \"s\": \"x,y} \\\\\"q\\\\\"\", \"flag\": false, \"none\": null, \"obj\": {\"B\": 1, \"a\": [1, \"t\"]}}', got '{\"flag\": false, \"key with space\": 1.5e-05, \"none\": \"null\", \"obj\": {\"a\": [1, \"t\"], \"B\": 1}, \"s\": \"x,y} \\\\\"q\\\\\"\"}'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"key with space\": 1.5e-05, \"s\": \"x,y} \\\\\"q\\\\\"\", \"flag\": false, \"none\": null, \"obj\": {\"B\": 1, \"a\": [1, \"t\"]}}', got '{\"flag\": false, \"key with space\": 1.5e-05, \"none\": \"null\", \"obj\": {\"a\": [1, \"t\"], \"B\": 1}, \"s\": \"x,y} \\\\\"q\\\\\"\"}'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "record", "arguments_raw": "{\"flag\": false, \"key with space\": 1.5e-05, \"none\": \"null\", \"obj\": {\"a\": [1, \"t\"], \"B\": 1}, \"s\": \"x,y} \\\"q\\\"\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "record", "arguments_raw": "{\"flag\": false, \"key with space\": 1.5e-05, \"none\": \"null\", \"obj\": {\"a\": [1, \"t\"], \"B\": 1}, \"s\": \"x,y} \\\"q\\\"\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "record", "arguments_raw": "{\"flag\": false, \"key with space\": 1.5e-05, \"none\": \"null\", \"obj\": {\"a\": [1, \"t\"], \"B\": 1}, \"s\": \"x,y} \\\"q\\\"\"}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "record", "arguments_raw": "{\"flag\": false, \"key with space\": 1.5e-05, \"none\": \"null\", \"obj\": {\"a\": [1, \"t\"], \"B\": 1}, \"s\": \"x,y} \\\"q\\\"\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "record", "arguments_raw": "{\"flag\": false, \"key with space\": 1.5e-05, \"none\": \"null\", \"obj\": {\"a\": [1, \"t\"], \"B\": 1}, \"s\": \"x,y} \\\"q\\\"\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "record", "arguments_raw": "{\"flag\": false, \"key with space\": 1.5e-05, \"none\": \"null\", \"obj\": {\"a\": [1, \"t\"], \"B\": 1}, \"s\": \"x,y} \\\"q\\\"\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "record", "arguments_raw": "{\"flag\": false, \"key with space\": 1.5e-05, \"none\": \"null\", \"obj\": {\"a\": [1, \"t\"], \"B\": 1}, \"s\": \"x,y} \\\"q\\\"\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "record", "arguments_raw": "{\"flag\": false, \"key with space\": 1.5e-05, \"none\": \"null\", \"obj\": {\"a\": [1, \"t\"], \"B\": 1}, \"s\": \"x,y} \\\"q\\\"\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "record", "arguments_raw": "{\"flag\": false, \"key with space\": 1.5e-05, \"none\": \"null\", \"obj\": {\"a\": [1, \"t\"], \"B\": 1}, \"s\": \"x,y} \\\"q\\\"\"}" } ], "exception": null } } } }, { "fixture_id": "gemma4/dictsort-case-insensitive-order", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gemma4/marker-in-arguments", "family": "gemma4", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}', got '{\"text\": \"Wrap calls as <|tool_call>call:f{}\"}'" }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"Wrap calls as <|tool_call>call:f{}\"}', got '{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"Wrap calls as <|tool_call>call:f{}\"}', got '{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"Wrap calls as <|tool_call>call:f{}\"}', got '{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"Wrap calls as <|tool_call>call:f{}\"}', got '{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"Wrap calls as <|tool_call>call:f{}\"}', got '{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"Wrap calls as <|tool_call>call:f{}\"}', got '{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"Wrap calls as <|tool_call>call:f{}\"}', got '{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"Wrap calls as <|tool_call>call:f{}\"}', got '{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}'" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Wrap calls as <|tool_call>call:f{}\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Wrap calls as <|tool_call>call:f{} and close thoughts with .\"}" } ], "exception": null } } } }, { "fixture_id": "gemma4/unicode-emoji", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gemma4/numeric-arguments", "family": "gemma4", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21]}', got '{\"exact\": true, \"limit\": \"null\", \"tolerance\": \"1e-05\", \"values\": [0, -0.5, \"1e+21\"], \"x\": -3, \"y\": 2.5}'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21]}', got '{\"exact\": true, \"limit\": \"null\", \"tolerance\": \"1e-05\", \"values\": [0, -0.5, \"1e+21\"], \"x\": -3, \"y\": 2.5}'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21]}', got '{\"exact\": true, \"limit\": \"null\", \"tolerance\": \"1e-05\", \"values\": [0, -0.5, \"1e+21\"], \"x\": -3, \"y\": 2.5}'" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21]}', got '{\"exact\": true, \"limit\": \"null\", \"tolerance\": \"1e-05\", \"values\": [0, -0.5, \"1e+21\"], \"x\": -3, \"y\": 2.5}'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21]}', got '{\"exact\": true, \"limit\": \"null\", \"tolerance\": \"1e-05\", \"values\": [0, -0.5, \"1e+21\"], \"x\": -3, \"y\": 2.5}'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21]}', got '{\"exact\": true, \"limit\": \"null\", \"tolerance\": \"1e-05\", \"values\": [0, -0.5, \"1e+21\"], \"x\": -3, \"y\": 2.5}'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21]}', got '{\"exact\": true, \"limit\": \"null\", \"tolerance\": \"1e-05\", \"values\": [0, -0.5, \"1e+21\"], \"x\": -3, \"y\": 2.5}'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21]}', got '{\"exact\": true, \"limit\": \"null\", \"tolerance\": \"1e-05\", \"values\": [0, -0.5, \"1e+21\"], \"x\": -3, \"y\": 2.5}'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21]}', got '{\"exact\": true, \"limit\": \"null\", \"tolerance\": \"1e-05\", \"values\": [0, -0.5, \"1e+21\"], \"x\": -3, \"y\": 2.5}'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "fail", "detail": "[0] calculate: $.limit: 'null' is not valid under any of the given schemas; $.tolerance: '1e-05' is not of type 'number'; $.values[2]: '1e+21' is not of type 'number'" }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] calculate: $.limit: 'null' is not valid under any of the given schemas; $.tolerance: '1e-05' is not of type 'number'; $.values[2]: '1e+21' is not of type 'number'" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] calculate: $.limit: 'null' is not valid under any of the given schemas; $.tolerance: '1e-05' is not of type 'number'; $.values[2]: '1e+21' is not of type 'number'" }, { "check": "arguments_schema", "strategy": "token", "status": "fail", "detail": "[0] calculate: $.limit: 'null' is not valid under any of the given schemas; $.tolerance: '1e-05' is not of type 'number'; $.values[2]: '1e+21' is not of type 'number'" }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "fail", "detail": "[0] calculate: $.limit: 'null' is not valid under any of the given schemas; $.tolerance: '1e-05' is not of type 'number'; $.values[2]: '1e+21' is not of type 'number'" }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "fail", "detail": "[0] calculate: $.limit: 'null' is not valid under any of the given schemas; $.tolerance: '1e-05' is not of type 'number'; $.values[2]: '1e+21' is not of type 'number'" }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "fail", "detail": "[0] calculate: $.limit: 'null' is not valid under any of the given schemas; $.tolerance: '1e-05' is not of type 'number'; $.values[2]: '1e+21' is not of type 'number'" }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[0] calculate: $.limit: 'null' is not valid under any of the given schemas; $.tolerance: '1e-05' is not of type 'number'; $.values[2]: '1e+21' is not of type 'number'" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "fail", "detail": "[0] calculate: $.limit: 'null' is not valid under any of the given schemas; $.tolerance: '1e-05' is not of type 'number'; $.values[2]: '1e+21' is not of type 'number'" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"exact\": true, \"limit\": \"null\", \"tolerance\": \"1e-05\", \"values\": [0, -0.5, \"1e+21\"], \"x\": -3, \"y\": 2.5}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"exact\": true, \"limit\": \"null\", \"tolerance\": \"1e-05\", \"values\": [0, -0.5, \"1e+21\"], \"x\": -3, \"y\": 2.5}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"exact\": true, \"limit\": \"null\", \"tolerance\": \"1e-05\", \"values\": [0, -0.5, \"1e+21\"], \"x\": -3, \"y\": 2.5}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"exact\": true, \"limit\": \"null\", \"tolerance\": \"1e-05\", \"values\": [0, -0.5, \"1e+21\"], \"x\": -3, \"y\": 2.5}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"exact\": true, \"limit\": \"null\", \"tolerance\": \"1e-05\", \"values\": [0, -0.5, \"1e+21\"], \"x\": -3, \"y\": 2.5}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"exact\": true, \"limit\": \"null\", \"tolerance\": \"1e-05\", \"values\": [0, -0.5, \"1e+21\"], \"x\": -3, \"y\": 2.5}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"exact\": true, \"limit\": \"null\", \"tolerance\": \"1e-05\", \"values\": [0, -0.5, \"1e+21\"], \"x\": -3, \"y\": 2.5}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"exact\": true, \"limit\": \"null\", \"tolerance\": \"1e-05\", \"values\": [0, -0.5, \"1e+21\"], \"x\": -3, \"y\": 2.5}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"exact\": true, \"limit\": \"null\", \"tolerance\": \"1e-05\", \"values\": [0, -0.5, \"1e+21\"], \"x\": -3, \"y\": 2.5}" } ], "exception": null } } } }, { "fixture_id": "gemma4/string-with-newlines-and-backslashes", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gemma4/json-text-in-string", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gemma4/array-of-strings", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gemma4/long-arguments", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gemma4/array-of-objects-with-braces", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gemma4/many-strings-then-string-array", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gemma4/vllm-no-tool-calls", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gemma4/vllm-single-call", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gemma4/vllm-multiple-arguments", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gemma4/vllm-text-before-call", "family": "gemma4", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you.', got 'Let me check the weather for you. '" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you.', got 'Let me check the weather for you. '" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you.', got 'Let me check the weather for you. '" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you.', got 'Let me check the weather for you. '" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you.', got 'Let me check the weather for you. '" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you.', got 'Let me check the weather for you. '" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you.', got 'Let me check the weather for you. '" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you.', got 'Let me check the weather for you. '" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you.', got 'Let me check the weather for you. '" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "Let me check the weather for you. ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Paris\"}" } ], "exception": null }, "streams": { "one": { "content": "Let me check the weather for you. ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Paris\"}" } ], "exception": null }, "special": { "content": "Let me check the weather for you. ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Paris\"}" } ], "exception": null }, "token": { "content": "Let me check the weather for you. ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Paris\"}" } ], "exception": null }, "rand:1:8": { "content": "Let me check the weather for you. ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Paris\"}" } ], "exception": null }, "rand:2:8": { "content": "Let me check the weather for you. ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Paris\"}" } ], "exception": null }, "rand:3:8": { "content": "Let me check the weather for you. ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Paris\"}" } ], "exception": null }, "rand:4:8": { "content": "Let me check the weather for you. ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Paris\"}" } ], "exception": null }, "rand:5:8": { "content": "Let me check the weather for you. ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Paris\"}" } ], "exception": null } } } }, { "fixture_id": "gemma4/vllm-two-calls", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gemma4/vllm-nested-arguments", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gemma4/vllm-number-and-boolean", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gemma4/vllm-hyphenated-function-name", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gemma4/vllm-dotted-function-name", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gemma4/vllm-no-arguments", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gemma4/vllm-reasoning-thought-prefix", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 105, 4368, 107 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gemma4/vllm-reasoning-thought-prefix-multiline", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 105, 4368, 107 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gemma4/sglang-text-before-call", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gemma4/sglang-text-around-call", "family": "gemma4", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected 'Some text before after', got 'Some text before '" }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream 'Some text before ', got 'Some text before after'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream 'Some text before ', got 'Some text before after'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream 'Some text before ', got 'Some text before after'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream 'Some text before ', got 'Some text before after'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream 'Some text before ', got 'Some text before after'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream 'Some text before ', got 'Some text before after'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream 'Some text before ', got 'Some text before after'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream 'Some text before ', got 'Some text before after'" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "Some text before ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Tokyo\"}" } ], "exception": null }, "streams": { "one": { "content": "Some text before after", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Tokyo\"}" } ], "exception": null }, "special": { "content": "Some text before after", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Tokyo\"}" } ], "exception": null }, "token": { "content": "Some text before after", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Tokyo\"}" } ], "exception": null }, "rand:1:8": { "content": "Some text before after", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Tokyo\"}" } ], "exception": null }, "rand:2:8": { "content": "Some text before after", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Tokyo\"}" } ], "exception": null }, "rand:3:8": { "content": "Some text before after", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Tokyo\"}" } ], "exception": null }, "rand:4:8": { "content": "Some text before after", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Tokyo\"}" } ], "exception": null }, "rand:5:8": { "content": "Some text before after", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Tokyo\"}" } ], "exception": null } } } }, { "fixture_id": "gemma4/sglang-nested-array-with-spaces", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gemma4/sglang-nested-object", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gemma4/vllm-malformed-no-brace", "family": "gemma4", "status": "fail", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '<|tool_call>call:bad_func no brace', got None" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '<|tool_call>call:bad_func no brace', got None" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '<|tool_call>call:bad_func no brace', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '<|tool_call>call:bad_func no brace', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '<|tool_call>call:bad_func no brace', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '<|tool_call>call:bad_func no brace', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '<|tool_call>call:bad_func no brace', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '<|tool_call>call:bad_func no brace', got None" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "<|tool_call>call:bad_func no brace", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null } } } }, { "fixture_id": "gemma4/multi-turn-final-answer", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "chat_template", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gemma4/multi-turn-second-call", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "chat_template", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gemma4/parallel-two-calls-thinking", "family": "gemma4", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 105, 4368, 107 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"filters\": {\"max\": 3, \"tags\": [\"a\", \"b\"]}, \"query\": \"café \\\"best\\\"\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"filters\": {\"max\": 3, \"tags\": [\"a\", \"b\"]}, \"query\": \"café \\\"best\\\"\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"filters\": {\"max\": 3, \"tags\": [\"a\", \"b\"]}, \"query\": \"café \\\"best\\\"\"}" } ], "exception": null }, "token": { "content": null, "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"filters\": {\"max\": 3, \"tags\": [\"a\", \"b\"]}, \"query\": \"café \\\"best\\\"\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"filters\": {\"max\": 3, \"tags\": [\"a\", \"b\"]}, \"query\": \"café \\\"best\\\"\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"filters\": {\"max\": 3, \"tags\": [\"a\", \"b\"]}, \"query\": \"café \\\"best\\\"\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"filters\": {\"max\": 3, \"tags\": [\"a\", \"b\"]}, \"query\": \"café \\\"best\\\"\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"filters\": {\"max\": 3, \"tags\": [\"a\", \"b\"]}, \"query\": \"café \\\"best\\\"\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"filters\": {\"max\": 3, \"tags\": [\"a\", \"b\"]}, \"query\": \"café \\\"best\\\"\"}" } ], "exception": null } } } }, { "fixture_id": "gemma4/parallel-three-calls", "family": "gemma4", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_weather', 'get_weather'], got ['get_weatherget_weatherget_weather']" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_weather', 'get_weather'], got ['get_weatherget_weatherget_weather']" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_weather', 'get_weather'], got ['get_weatherget_weatherget_weather']" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_weather', 'get_weather'], got ['get_weatherget_weatherget_weather']" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_weather', 'get_weather'], got ['get_weatherget_weatherget_weather']" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_weather', 'get_weather'], got ['get_weatherget_weatherget_weather']" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_weather', 'get_weather'], got ['get_weatherget_weatherget_weather']" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_weather', 'get_weather'], got ['get_weatherget_weatherget_weather']" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'get_weather', 'get_weather'], got ['get_weatherget_weatherget_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'get_weather', 'get_weather'], got ['get_weatherget_weatherget_weather']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'get_weather', 'get_weather'], got ['get_weatherget_weatherget_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'get_weather', 'get_weather'], got ['get_weatherget_weatherget_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'get_weather', 'get_weather'], got ['get_weatherget_weatherget_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'get_weather', 'get_weather'], got ['get_weatherget_weatherget_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'get_weather', 'get_weather'], got ['get_weatherget_weatherget_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'get_weather', 'get_weather'], got ['get_weatherget_weatherget_weather']" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_weatherget_weatherget_weather: arguments are not valid JSON text ('{\"city\": \"Paris\"}{\"city\": \"London\"}{\"city\": \"Tokyo\", \"unit\": \"c\"}': Extra data: line 1 column 18 (char 17))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_weatherget_weatherget_weather: arguments are not valid JSON text ('{\"city\": \"Paris\"}{\"city\": \"London\"}{\"city\": \"Tokyo\", \"unit\": \"c\"}': Extra data: line 1 column 18 (char 17))" }, { "check": "arguments_json", "strategy": "token", "status": "fail", "detail": "[0] get_weatherget_weatherget_weather: arguments are not valid JSON text ('{\"city\": \"Paris\"}{\"city\": \"London\"}{\"city\": \"Tokyo\", \"unit\": \"c\"}': Extra data: line 1 column 18 (char 17))" }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "fail", "detail": "[0] get_weatherget_weatherget_weather: arguments are not valid JSON text ('{\"city\": \"Paris\"}{\"city\": \"London\"}{\"city\": \"Tokyo\", \"unit\": \"c\"}': Extra data: line 1 column 18 (char 17))" }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "fail", "detail": "[0] get_weatherget_weatherget_weather: arguments are not valid JSON text ('{\"city\": \"Paris\"}{\"city\": \"London\"}{\"city\": \"Tokyo\", \"unit\": \"c\"}': Extra data: line 1 column 18 (char 17))" }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "fail", "detail": "[0] get_weatherget_weatherget_weather: arguments are not valid JSON text ('{\"city\": \"Paris\"}{\"city\": \"London\"}{\"city\": \"Tokyo\", \"unit\": \"c\"}': Extra data: line 1 column 18 (char 17))" }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "fail", "detail": "[0] get_weatherget_weatherget_weather: arguments are not valid JSON text ('{\"city\": \"Paris\"}{\"city\": \"London\"}{\"city\": \"Tokyo\", \"unit\": \"c\"}': Extra data: line 1 column 18 (char 17))" }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "fail", "detail": "[0] get_weatherget_weatherget_weather: arguments are not valid JSON text ('{\"city\": \"Paris\"}{\"city\": \"London\"}{\"city\": \"Tokyo\", \"unit\": \"c\"}': Extra data: line 1 column 18 (char 17))" }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] 'get_weatherget_weatherget_weather' is not an offered tool" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] 'get_weatherget_weatherget_weather' is not an offered tool" }, { "check": "arguments_schema", "strategy": "token", "status": "fail", "detail": "[0] 'get_weatherget_weatherget_weather' is not an offered tool" }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "fail", "detail": "[0] 'get_weatherget_weatherget_weather' is not an offered tool" }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "fail", "detail": "[0] 'get_weatherget_weatherget_weather' is not an offered tool" }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "fail", "detail": "[0] 'get_weatherget_weatherget_weather' is not an offered tool" }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[0] 'get_weatherget_weatherget_weather' is not an offered tool" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "fail", "detail": "[0] 'get_weatherget_weatherget_weather' is not an offered tool" }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "fail", "detail": "expected 3 calls ['get_weather', 'get_weather', 'get_weather'], got 1 ['get_weatherget_weatherget_weather']" }, { "check": "parallel_order", "strategy": "special", "status": "fail", "detail": "expected 3 calls ['get_weather', 'get_weather', 'get_weather'], got 1 ['get_weatherget_weatherget_weather']" }, { "check": "parallel_order", "strategy": "token", "status": "fail", "detail": "expected 3 calls ['get_weather', 'get_weather', 'get_weather'], got 1 ['get_weatherget_weatherget_weather']" }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "fail", "detail": "expected 3 calls ['get_weather', 'get_weather', 'get_weather'], got 1 ['get_weatherget_weatherget_weather']" }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "fail", "detail": "expected 3 calls ['get_weather', 'get_weather', 'get_weather'], got 1 ['get_weatherget_weatherget_weather']" }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "fail", "detail": "expected 3 calls ['get_weather', 'get_weather', 'get_weather'], got 1 ['get_weatherget_weatherget_weather']" }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "fail", "detail": "expected 3 calls ['get_weather', 'get_weather', 'get_weather'], got 1 ['get_weatherget_weatherget_weather']" }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "fail", "detail": "expected 3 calls ['get_weather', 'get_weather', 'get_weather'], got 1 ['get_weatherget_weatherget_weather']" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"c\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weatherget_weatherget_weather", "arguments_raw": "{\"city\": \"Paris\"}{\"city\": \"London\"}{\"city\": \"Tokyo\", \"unit\": \"c\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weatherget_weatherget_weather", "arguments_raw": "{\"city\": \"Paris\"}{\"city\": \"London\"}{\"city\": \"Tokyo\", \"unit\": \"c\"}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weatherget_weatherget_weather", "arguments_raw": "{\"city\": \"Paris\"}{\"city\": \"London\"}{\"city\": \"Tokyo\", \"unit\": \"c\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weatherget_weatherget_weather", "arguments_raw": "{\"city\": \"Paris\"}{\"city\": \"London\"}{\"city\": \"Tokyo\", \"unit\": \"c\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weatherget_weatherget_weather", "arguments_raw": "{\"city\": \"Paris\"}{\"city\": \"London\"}{\"city\": \"Tokyo\", \"unit\": \"c\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weatherget_weatherget_weather", "arguments_raw": "{\"city\": \"Paris\"}{\"city\": \"London\"}{\"city\": \"Tokyo\", \"unit\": \"c\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weatherget_weatherget_weather", "arguments_raw": "{\"city\": \"Paris\"}{\"city\": \"London\"}{\"city\": \"Tokyo\", \"unit\": \"c\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weatherget_weatherget_weather", "arguments_raw": "{\"city\": \"Paris\"}{\"city\": \"London\"}{\"city\": \"Tokyo\", \"unit\": \"c\"}" } ], "exception": null } } } }, { "fixture_id": "gemma4/parallel-mixed-with-empty", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": "", "id": 1, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gemma4/truncated-mid-string", "family": "gemma4", "status": "fail", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists." }, { "check": "expected_error", "strategy": "special", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists." }, { "check": "expected_error", "strategy": "token", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists." }, { "check": "expected_error", "strategy": "rand:1:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists." }, { "check": "expected_error", "strategy": "rand:2:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists." }, { "check": "expected_error", "strategy": "rand:3:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists." }, { "check": "expected_error", "strategy": "rand:4:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists." }, { "check": "expected_error", "strategy": "rand:5:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists." }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '<|tool_call>call:get_weather{city:<|\"|>Paris', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '<|tool_call>call:get_weather{city:<|\"|>Paris', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '<|tool_call>call:get_weather{city:<|\"|>Paris', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '<|tool_call>call:get_weather{city:<|\"|>Paris', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '<|tool_call>call:get_weather{city:<|\"|>Paris', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '<|tool_call>call:get_weather{city:<|\"|>Paris', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '<|tool_call>call:get_weather{city:<|\"|>Paris', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '<|tool_call>call:get_weather{city:<|\"|>Paris', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "<|tool_call>call:get_weather{city:<|\"|>Paris", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null } } } }, { "fixture_id": "gemma4/truncated-in-name", "family": "gemma4", "status": "fail", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '<|tool_call>call:get', got None" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '<|tool_call>call:get', got None" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '<|tool_call>call:get', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '<|tool_call>call:get', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '<|tool_call>call:get', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '<|tool_call>call:get', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '<|tool_call>call:get', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '<|tool_call>call:get', got None" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "<|tool_call>call:get", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null } } } }, { "fixture_id": "gemma4/truncated-before-close", "family": "gemma4", "status": "fail", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists." }, { "check": "expected_error", "strategy": "special", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists." }, { "check": "expected_error", "strategy": "token", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists." }, { "check": "expected_error", "strategy": "rand:1:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists." }, { "check": "expected_error", "strategy": "rand:2:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists." }, { "check": "expected_error", "strategy": "rand:3:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists." }, { "check": "expected_error", "strategy": "rand:4:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists." }, { "check": "expected_error", "strategy": "rand:5:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool call: no complete call exists." }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '<|tool_call>call:get_weather{city:<|\"|>Paris<|\"|>', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '<|tool_call>call:get_weather{city:<|\"|>Paris<|\"|>', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '<|tool_call>call:get_weather{city:<|\"|>Paris<|\"|>', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '<|tool_call>call:get_weather{city:<|\"|>Paris<|\"|>', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '<|tool_call>call:get_weather{city:<|\"|>Paris<|\"|>', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '<|tool_call>call:get_weather{city:<|\"|>Paris<|\"|>', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '<|tool_call>call:get_weather{city:<|\"|>Paris<|\"|>', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '<|tool_call>call:get_weather{city:<|\"|>Paris<|\"|>', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "<|tool_call>call:get_weather{city:<|\"|>Paris<|\"|>", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null } } } }, { "fixture_id": "gemma4/truncated-in-reasoning", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 105, 4368, 107 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gemma4/truncated-second-parallel-call", "family": "gemma4", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream ['get_weather'], got ['get_weather', 'search']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls: nonstream ['get_weather'], got ['get_weather', 'search']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "tool_calls: nonstream ['get_weather'], got ['get_weather', 'search']" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: nonstream ['get_weather'], got ['get_weather', 'search']" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: nonstream ['get_weather'], got ['get_weather', 'search']" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: nonstream ['get_weather'], got ['get_weather', 'search']" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: nonstream ['get_weather'], got ['get_weather', 'search']" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: nonstream ['get_weather'], got ['get_weather', 'search']" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[1] search: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[1] search: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "fail", "detail": "[1] search: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "fail", "detail": "[1] search: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "fail", "detail": "[1] search: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "fail", "detail": "[1] search: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "fail", "detail": "[1] search: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "fail", "detail": "[1] search: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[1] search: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[1] search: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "fail", "detail": "[1] search: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "fail", "detail": "[1] search: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "fail", "detail": "[1] search: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "fail", "detail": "[1] search: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[1] search: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "fail", "detail": "[1] search: arguments are not a JSON object; not validated" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gemma4", "reasoning_parser": "gemma4", "tool_call_detector": "Gemma4Detector", "reasoning_detector": "Gemma4Detector", "auto_detected": { "tool_call_parser": "gemma4", "reasoning_parser": "gemma4" }, "model": "google/gemma-4-31B-it", "tokenizer": { "repo": "google/gemma-4-31B-it", "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475", "class": "GemmaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Gemma4ForConditionalGeneration" ], "model_type": "gemma4" }, "hf_config_error": null, "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=False, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 107, 100, 45518, 107, 101 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "" } ], "exception": null } } } }, { "fixture_id": "glm/glm45-single-call", "family": "glm", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "reasoning_content: expected 'The user wants the current weather in Paris, in Celsius.', got '\\nThe user wants the current weather in Paris, in Celsius.'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user wants the current weather in Paris, in Celsius.', got '\\nThe user wants the current weather in Paris, in Celsius.'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user wants the current weather in Paris, in Celsius.', got '\\nThe user wants the current weather in Paris, in Celsius.'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'; reasoning_content (whitespace only): expected 'The user wants the current weather in Paris, in Celsius.', got '\\nThe user wants the current weather in Paris, in Celsius.'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user wants the current weather in Paris, in Celsius.', got '\\nThe user wants the current weather in Paris, in Celsius.'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user wants the current weather in Paris, in Celsius.', got '\\nThe user wants the current weather in Paris, in Celsius.'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user wants the current weather in Paris, in Celsius.', got '\\nThe user wants the current weather in Paris, in Celsius.'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user wants the current weather in Paris, in Celsius.', got '\\nThe user wants the current weather in Paris, in Celsius.'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user wants the current weather in Paris, in Celsius.', got '\\nThe user wants the current weather in Paris, in Celsius.'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "reasoning_content: nonstream '\\nThe user wants the current weather in Paris, in Celsius.', got '\\nThe user wants the current weather in Paris, in Celsius.'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "reasoning_content: nonstream '\\nThe user wants the current weather in Paris, in Celsius.', got '\\nThe user wants the current weather in Paris, in Celsius.'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content (whitespace only): nonstream None, got '\\n'; reasoning_content: nonstream '\\nThe user wants the current weather in Paris, in Celsius.', got '\\nThe user wants the current weather in Paris, in Celsius.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "reasoning_content: nonstream '\\nThe user wants the current weather in Paris, in Celsius.', got '\\nThe user wants the current weather in Paris, in Celsius.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "reasoning_content: nonstream '\\nThe user wants the current weather in Paris, in Celsius.', got '\\nThe user wants the current weather in Paris, in Celsius.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "reasoning_content: nonstream '\\nThe user wants the current weather in Paris, in Celsius.', got '\\nThe user wants the current weather in Paris, in Celsius.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "reasoning_content: nonstream '\\nThe user wants the current weather in Paris, in Celsius.', got '\\nThe user wants the current weather in Paris, in Celsius.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "reasoning_content: nonstream '\\nThe user wants the current weather in Paris, in Celsius.', got '\\nThe user wants the current weather in Paris, in Celsius.'" }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one None, got '\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "reasoning_content contains ''" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm45", "reasoning_parser": "glm45", "tool_call_detector": "Glm4MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm45", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.5", "tokenizer": { "repo": "zai-org/GLM-4.5", "revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "\nThe user wants the current weather in Paris, in Celsius.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "\nThe user wants the current weather in Paris, in Celsius.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "\nThe user wants the current weather in Paris, in Celsius.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}" } ], "exception": null }, "token": { "content": "\n", "reasoning_content": "\nThe user wants the current weather in Paris, in Celsius.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "\nThe user wants the current weather in Paris, in Celsius.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "\nThe user wants the current weather in Paris, in Celsius.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "\nThe user wants the current weather in Paris, in Celsius.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "\nThe user wants the current weather in Paris, in Celsius.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "\nThe user wants the current weather in Paris, in Celsius.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}" } ], "exception": null } } } }, { "fixture_id": "glm/glm45-parallel-same-tool", "family": "glm", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "reasoning_content: expected 'Two cities, so two independent weather lookups.', got '\\nTwo cities, so two independent weather lookups.'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "reasoning_content (whitespace only): expected 'Two cities, so two independent weather lookups.', got '\\nTwo cities, so two independent weather lookups.'; tool_calls: expected ['get_weather', 'get_weather'], got ['get_weather']" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "reasoning_content (whitespace only): expected 'Two cities, so two independent weather lookups.', got '\\nTwo cities, so two independent weather lookups.'; tool_calls: expected ['get_weather', 'get_weather'], got ['get_weather']" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Two cities, so two independent weather lookups.', got '\\nTwo cities, so two independent weather lookups.'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Two cities, so two independent weather lookups.', got '\\nTwo cities, so two independent weather lookups.'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'; reasoning_content (whitespace only): expected 'Two cities, so two independent weather lookups.', got '\\nTwo cities, so two independent weather lookups.'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Two cities, so two independent weather lookups.', got '\\nTwo cities, so two independent weather lookups.'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Two cities, so two independent weather lookups.', got '\\nTwo cities, so two independent weather lookups.'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Two cities, so two independent weather lookups.', got '\\nTwo cities, so two independent weather lookups.'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "reasoning_content: nonstream '\\nTwo cities, so two independent weather lookups.', got '\\nTwo cities, so two independent weather lookups.'; tool_calls: nonstream ['get_weather', 'get_weather'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "reasoning_content: nonstream '\\nTwo cities, so two independent weather lookups.', got '\\nTwo cities, so two independent weather lookups.'; tool_calls: nonstream ['get_weather', 'get_weather'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content (whitespace only): nonstream None, got '\\n\\n'; reasoning_content: nonstream '\\nTwo cities, so two independent weather lookups.', got '\\nTwo cities, so two independent weather lookups.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "reasoning_content: nonstream '\\nTwo cities, so two independent weather lookups.', got '\\nTwo cities, so two independent weather lookups.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content (whitespace only): nonstream None, got '\\n'; reasoning_content: nonstream '\\nTwo cities, so two independent weather lookups.', got '\\nTwo cities, so two independent weather lookups.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "reasoning_content: nonstream '\\nTwo cities, so two independent weather lookups.', got '\\nTwo cities, so two independent weather lookups.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "reasoning_content: nonstream '\\nTwo cities, so two independent weather lookups.', got '\\nTwo cities, so two independent weather lookups.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "reasoning_content: nonstream '\\nTwo cities, so two independent weather lookups.', got '\\nTwo cities, so two independent weather lookups.'" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content (whitespace only): one None, got '\\n\\n'; tool_calls: one ['get_weather'], got ['get_weather', 'get_weather'] | rand:1:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'get_weather'] | rand:2:8 (fail): content (whitespace only): one None, got '\\n'; tool_calls: one ['get_weather'], got ['get_weather', 'get_weather'] | rand:3:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'get_weather'] | rand:4:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'get_weather'] | rand:5:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'get_weather']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "reasoning_content contains ''" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "fail", "detail": "expected 2 calls ['get_weather', 'get_weather'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "special", "status": "fail", "detail": "expected 2 calls ['get_weather', 'get_weather'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm45", "reasoning_parser": "glm45", "tool_call_detector": "Glm4MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm45", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.5", "tokenizer": { "repo": "zai-org/GLM-4.5", "revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "\nTwo cities, so two independent weather lookups.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "\nTwo cities, so two independent weather lookups.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "\nTwo cities, so two independent weather lookups.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "token": { "content": "\n\n", "reasoning_content": "\nTwo cities, so two independent weather lookups.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "\nTwo cities, so two independent weather lookups.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null }, "rand:2:8": { "content": "\n", "reasoning_content": "\nTwo cities, so two independent weather lookups.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "\nTwo cities, so two independent weather lookups.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "\nTwo cities, so two independent weather lookups.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "\nTwo cities, so two independent weather lookups.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null } } } }, { "fixture_id": "glm/glm45-parallel-mixed-nested", "family": "glm", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "reasoning_content: expected 'Check the weather, then search for cafés.', got '\\nCheck the weather, then search for cafés.'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "reasoning_content (whitespace only): expected 'Check the weather, then search for cafés.', got '\\nCheck the weather, then search for cafés.'; tool_calls: expected ['get_weather', 'search_web'], got ['get_weather']" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "reasoning_content (whitespace only): expected 'Check the weather, then search for cafés.', got '\\nCheck the weather, then search for cafés.'; tool_calls: expected ['get_weather', 'search_web'], got ['get_weather']" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Check the weather, then search for cafés.', got '\\nCheck the weather, then search for cafés.'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Check the weather, then search for cafés.', got '\\nCheck the weather, then search for cafés.'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'; reasoning_content (whitespace only): expected 'Check the weather, then search for cafés.', got '\\nCheck the weather, then search for cafés.'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Check the weather, then search for cafés.', got '\\nCheck the weather, then search for cafés.'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Check the weather, then search for cafés.', got '\\nCheck the weather, then search for cafés.'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Check the weather, then search for cafés.', got '\\nCheck the weather, then search for cafés.'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "reasoning_content: nonstream '\\nCheck the weather, then search for cafés.', got '\\nCheck the weather, then search for cafés.'; tool_calls: nonstream ['get_weather', 'search_web'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "reasoning_content: nonstream '\\nCheck the weather, then search for cafés.', got '\\nCheck the weather, then search for cafés.'; tool_calls: nonstream ['get_weather', 'search_web'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content (whitespace only): nonstream None, got '\\n\\n'; reasoning_content: nonstream '\\nCheck the weather, then search for cafés.', got '\\nCheck the weather, then search for cafés.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "reasoning_content: nonstream '\\nCheck the weather, then search for cafés.', got '\\nCheck the weather, then search for cafés.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content (whitespace only): nonstream None, got '\\n'; reasoning_content: nonstream '\\nCheck the weather, then search for cafés.', got '\\nCheck the weather, then search for cafés.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "reasoning_content: nonstream '\\nCheck the weather, then search for cafés.', got '\\nCheck the weather, then search for cafés.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "reasoning_content: nonstream '\\nCheck the weather, then search for cafés.', got '\\nCheck the weather, then search for cafés.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "reasoning_content: nonstream '\\nCheck the weather, then search for cafés.', got '\\nCheck the weather, then search for cafés.'" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content (whitespace only): one None, got '\\n\\n'; tool_calls: one ['get_weather'], got ['get_weather', 'search_web'] | rand:1:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search_web'] | rand:2:8 (fail): content (whitespace only): one None, got '\\n'; tool_calls: one ['get_weather'], got ['get_weather', 'search_web'] | rand:3:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search_web'] | rand:4:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search_web'] | rand:5:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search_web']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "reasoning_content contains ''" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "fail", "detail": "expected 2 calls ['get_weather', 'search_web'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "special", "status": "fail", "detail": "expected 2 calls ['get_weather', 'search_web'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm45", "reasoning_parser": "glm45", "tool_call_detector": "Glm4MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm45", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.5", "tokenizer": { "repo": "zai-org/GLM-4.5", "revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "\nCheck the weather, then search for cafés.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"celsius\"}" }, { "name": "search_web", "arguments_raw": "{\"query\": \"best cafés\", \"filters\": {\"site\": \"example.com\", \"max_results\": 5, \"tags\": [\"coffee\", \"wifi\"]}}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "\nCheck the weather, then search for cafés.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"celsius\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "\nCheck the weather, then search for cafés.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"celsius\"}" } ], "exception": null }, "token": { "content": "\n\n", "reasoning_content": "\nCheck the weather, then search for cafés.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"celsius\"}" }, { "name": "search_web", "arguments_raw": "{\"query\": \"best cafés\", \"filters\": {\"site\": \"example.com\", \"max_results\": 5, \"tags\": [\"coffee\", \"wifi\"]}}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "\nCheck the weather, then search for cafés.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"celsius\"}" }, { "name": "search_web", "arguments_raw": "{\"query\": \"best cafés\", \"filters\": {\"site\": \"example.com\", \"max_results\": 5, \"tags\": [\"coffee\", \"wifi\"]}}" } ], "exception": null }, "rand:2:8": { "content": "\n", "reasoning_content": "\nCheck the weather, then search for cafés.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"celsius\"}" }, { "name": "search_web", "arguments_raw": "{\"query\": \"best cafés\", \"filters\": {\"site\": \"example.com\", \"max_results\": 5, \"tags\": [\"coffee\", \"wifi\"]}}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "\nCheck the weather, then search for cafés.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"celsius\"}" }, { "name": "search_web", "arguments_raw": "{\"query\": \"best cafés\", \"filters\": {\"site\": \"example.com\", \"max_results\": 5, \"tags\": [\"coffee\", \"wifi\"]}}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "\nCheck the weather, then search for cafés.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"celsius\"}" }, { "name": "search_web", "arguments_raw": "{\"query\": \"best cafés\", \"filters\": {\"site\": \"example.com\", \"max_results\": 5, \"tags\": [\"coffee\", \"wifi\"]}}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "\nCheck the weather, then search for cafés.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"celsius\"}" }, { "name": "search_web", "arguments_raw": "{\"query\": \"best cafés\", \"filters\": {\"site\": \"example.com\", \"max_results\": 5, \"tags\": [\"coffee\", \"wifi\"]}}" } ], "exception": null } } } }, { "fixture_id": "glm/glm45-content-before-call", "family": "glm", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "reasoning_content: expected 'Need live data, so call the tool.', got '\\nNeed live data, so call the tool.'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected \"I'll look that up for you.\", got None; reasoning_content (whitespace only): expected 'Need live data, so call the tool.', got '\\nNeed live data, so call the tool.'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected \"I'll look that up for you.\", got None; reasoning_content (whitespace only): expected 'Need live data, so call the tool.', got '\\nNeed live data, so call the tool.'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected \"I'll look that up for you.\", got \"\\nI'll look that up for you.\\n\"; reasoning_content (whitespace only): expected 'Need live data, so call the tool.', got '\\nNeed live data, so call the tool.'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected \"I'll look that up for you.\", got \"\\nI'll look that up for you\"; reasoning_content (whitespace only): expected 'Need live data, so call the tool.', got '\\nNeed live data, so call the tool.'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected \"I'll look that up for you.\", got \"\\nI'll look that up for\"; reasoning_content (whitespace only): expected 'Need live data, so call the tool.', got '\\nNeed live data, so call the tool.'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected \"I'll look that up for you.\", got \"\\nI'll look that up for you.\\n\"; reasoning_content (whitespace only): expected 'Need live data, so call the tool.', got '\\nNeed live data, so call the tool.'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected \"I'll look that up for you.\", got \"\\nI'll look that up\"; reasoning_content (whitespace only): expected 'Need live data, so call the tool.', got '\\nNeed live data, so call the tool.'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected \"I'll look that up for you.\", got \"\\nI'll look that up for you\"; reasoning_content (whitespace only): expected 'Need live data, so call the tool.', got '\\nNeed live data, so call the tool.'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream \"I'll look that up for you.\", got None; reasoning_content: nonstream '\\nNeed live data, so call the tool.', got '\\nNeed live data, so call the tool.'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream \"I'll look that up for you.\", got None; reasoning_content: nonstream '\\nNeed live data, so call the tool.', got '\\nNeed live data, so call the tool.'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content (whitespace only): nonstream \"I'll look that up for you.\", got \"\\nI'll look that up for you.\\n\"; reasoning_content: nonstream '\\nNeed live data, so call the tool.', got '\\nNeed live data, so call the tool.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream \"I'll look that up for you.\", got \"\\nI'll look that up for you\"; reasoning_content: nonstream '\\nNeed live data, so call the tool.', got '\\nNeed live data, so call the tool.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream \"I'll look that up for you.\", got \"\\nI'll look that up for\"; reasoning_content: nonstream '\\nNeed live data, so call the tool.', got '\\nNeed live data, so call the tool.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content (whitespace only): nonstream \"I'll look that up for you.\", got \"\\nI'll look that up for you.\\n\"; reasoning_content: nonstream '\\nNeed live data, so call the tool.', got '\\nNeed live data, so call the tool.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream \"I'll look that up for you.\", got \"\\nI'll look that up\"; reasoning_content: nonstream '\\nNeed live data, so call the tool.', got '\\nNeed live data, so call the tool.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream \"I'll look that up for you.\", got \"\\nI'll look that up for you\"; reasoning_content: nonstream '\\nNeed live data, so call the tool.', got '\\nNeed live data, so call the tool.'" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content: one None, got \"\\nI'll look that up for you.\\n\" | rand:1:8 (fail): content: one None, got \"\\nI'll look that up for you\" | rand:2:8 (fail): content: one None, got \"\\nI'll look that up for\" | rand:3:8 (fail): content: one None, got \"\\nI'll look that up for you.\\n\" | rand:4:8 (fail): content: one None, got \"\\nI'll look that up\" | rand:5:8 (fail): content: one None, got \"\\nI'll look that up for you\"" }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "reasoning_content contains ''" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm45", "reasoning_parser": "glm45", "tool_call_detector": "Glm4MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm45", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.5", "tokenizer": { "repo": "zai-org/GLM-4.5", "revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "I'll look that up for you.", "reasoning_content": "\nNeed live data, so call the tool.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "\nNeed live data, so call the tool.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "\nNeed live data, so call the tool.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\"}" } ], "exception": null }, "token": { "content": "\nI'll look that up for you.\n", "reasoning_content": "\nNeed live data, so call the tool.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\"}" } ], "exception": null }, "rand:1:8": { "content": "\nI'll look that up for you", "reasoning_content": "\nNeed live data, so call the tool.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\"}" } ], "exception": null }, "rand:2:8": { "content": "\nI'll look that up for", "reasoning_content": "\nNeed live data, so call the tool.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\"}" } ], "exception": null }, "rand:3:8": { "content": "\nI'll look that up for you.\n", "reasoning_content": "\nNeed live data, so call the tool.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\"}" } ], "exception": null }, "rand:4:8": { "content": "\nI'll look that up", "reasoning_content": "\nNeed live data, so call the tool.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\"}" } ], "exception": null }, "rand:5:8": { "content": "\nI'll look that up for you", "reasoning_content": "\nNeed live data, so call the tool.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\"}" } ], "exception": null } } } }, { "fixture_id": "glm/glm45-no-call", "family": "glm", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content (whitespace only): expected 'Paris is the capital of France.', got '\\nParis is the capital of France.'; reasoning_content: expected 'A simple fact; no tool is needed.', got '\\nA simple fact; no tool is needed.'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected 'Paris is the capital of France.', got '\\nParis is the capital of France.'; reasoning_content (whitespace only): expected 'A simple fact; no tool is needed.', got '\\nA simple fact; no tool is needed.'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected 'Paris is the capital of France.', got '\\nParis is the capital of France.'; reasoning_content (whitespace only): expected 'A simple fact; no tool is needed.', got '\\nA simple fact; no tool is needed.'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected 'Paris is the capital of France.', got '\\nParis is the capital of France.'; reasoning_content (whitespace only): expected 'A simple fact; no tool is needed.', got '\\nA simple fact; no tool is needed.'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Paris is the capital of France.', got '\\nParis is the capital of France.'; reasoning_content (whitespace only): expected 'A simple fact; no tool is needed.', got '\\nA simple fact; no tool is needed.'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Paris is the capital of France.', got '\\nParis is the capital of France.'; reasoning_content (whitespace only): expected 'A simple fact; no tool is needed.', got '\\nA simple fact; no tool is needed.'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Paris is the capital of France.', got '\\nParis is the capital of France.'; reasoning_content (whitespace only): expected 'A simple fact; no tool is needed.', got '\\nA simple fact; no tool is needed.'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Paris is the capital of France.', got '\\nParis is the capital of France.'; reasoning_content (whitespace only): expected 'A simple fact; no tool is needed.', got '\\nA simple fact; no tool is needed.'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Paris is the capital of France.', got '\\nParis is the capital of France.'; reasoning_content (whitespace only): expected 'A simple fact; no tool is needed.', got '\\nA simple fact; no tool is needed.'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "reasoning_content: nonstream '\\nA simple fact; no tool is needed.', got '\\nA simple fact; no tool is needed.'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "reasoning_content: nonstream '\\nA simple fact; no tool is needed.', got '\\nA simple fact; no tool is needed.'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "reasoning_content: nonstream '\\nA simple fact; no tool is needed.', got '\\nA simple fact; no tool is needed.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "reasoning_content: nonstream '\\nA simple fact; no tool is needed.', got '\\nA simple fact; no tool is needed.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "reasoning_content: nonstream '\\nA simple fact; no tool is needed.', got '\\nA simple fact; no tool is needed.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "reasoning_content: nonstream '\\nA simple fact; no tool is needed.', got '\\nA simple fact; no tool is needed.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "reasoning_content: nonstream '\\nA simple fact; no tool is needed.', got '\\nA simple fact; no tool is needed.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "reasoning_content: nonstream '\\nA simple fact; no tool is needed.', got '\\nA simple fact; no tool is needed.'" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "reasoning_content contains ''" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm45", "reasoning_parser": "glm45", "tool_call_detector": "Glm4MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm45", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.5", "tokenizer": { "repo": "zai-org/GLM-4.5", "revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "\nParis is the capital of France.", "reasoning_content": "\nA simple fact; no tool is needed.", "tool_calls": [], "exception": null }, "streams": { "one": { "content": "\nParis is the capital of France.", "reasoning_content": "\nA simple fact; no tool is needed.", "tool_calls": [], "exception": null }, "special": { "content": "\nParis is the capital of France.", "reasoning_content": "\nA simple fact; no tool is needed.", "tool_calls": [], "exception": null }, "token": { "content": "\nParis is the capital of France.", "reasoning_content": "\nA simple fact; no tool is needed.", "tool_calls": [], "exception": null }, "rand:1:8": { "content": "\nParis is the capital of France.", "reasoning_content": "\nA simple fact; no tool is needed.", "tool_calls": [], "exception": null }, "rand:2:8": { "content": "\nParis is the capital of France.", "reasoning_content": "\nA simple fact; no tool is needed.", "tool_calls": [], "exception": null }, "rand:3:8": { "content": "\nParis is the capital of France.", "reasoning_content": "\nA simple fact; no tool is needed.", "tool_calls": [], "exception": null }, "rand:4:8": { "content": "\nParis is the capital of France.", "reasoning_content": "\nA simple fact; no tool is needed.", "tool_calls": [], "exception": null }, "rand:5:8": { "content": "\nParis is the capital of France.", "reasoning_content": "\nA simple fact; no tool is needed.", "tool_calls": [], "exception": null } } } }, { "fixture_id": "glm/glm45-nested-object", "family": "glm", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "reasoning_content: expected 'Apply the dark theme and disable SMS.', got '\\nApply the dark theme and disable SMS.'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Apply the dark theme and disable SMS.', got '\\nApply the dark theme and disable SMS.'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Apply the dark theme and disable SMS.', got '\\nApply the dark theme and disable SMS.'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'; reasoning_content (whitespace only): expected 'Apply the dark theme and disable SMS.', got '\\nApply the dark theme and disable SMS.'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'; reasoning_content (whitespace only): expected 'Apply the dark theme and disable SMS.', got '\\nApply the dark theme and disable SMS.'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Apply the dark theme and disable SMS.', got '\\nApply the dark theme and disable SMS.'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Apply the dark theme and disable SMS.', got '\\nApply the dark theme and disable SMS.'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Apply the dark theme and disable SMS.', got '\\nApply the dark theme and disable SMS.'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'; reasoning_content (whitespace only): expected 'Apply the dark theme and disable SMS.', got '\\nApply the dark theme and disable SMS.'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "reasoning_content: nonstream '\\nApply the dark theme and disable SMS.', got '\\nApply the dark theme and disable SMS.'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "reasoning_content: nonstream '\\nApply the dark theme and disable SMS.', got '\\nApply the dark theme and disable SMS.'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content (whitespace only): nonstream None, got '\\n'; reasoning_content: nonstream '\\nApply the dark theme and disable SMS.', got '\\nApply the dark theme and disable SMS.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content (whitespace only): nonstream None, got '\\n'; reasoning_content: nonstream '\\nApply the dark theme and disable SMS.', got '\\nApply the dark theme and disable SMS.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "reasoning_content: nonstream '\\nApply the dark theme and disable SMS.', got '\\nApply the dark theme and disable SMS.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "reasoning_content: nonstream '\\nApply the dark theme and disable SMS.', got '\\nApply the dark theme and disable SMS.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "reasoning_content: nonstream '\\nApply the dark theme and disable SMS.', got '\\nApply the dark theme and disable SMS.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content (whitespace only): nonstream None, got '\\n'; reasoning_content: nonstream '\\nApply the dark theme and disable SMS.', got '\\nApply the dark theme and disable SMS.'" }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one None, got '\\n' | rand:1:8 (soft_pass): content (whitespace only): one None, got '\\n' | rand:5:8 (soft_pass): content (whitespace only): one None, got '\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "reasoning_content contains ''" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm45", "reasoning_parser": "glm45", "tool_call_detector": "Glm4MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm45", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.5", "tokenizer": { "repo": "zai-org/GLM-4.5", "revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "\nApply the dark theme and disable SMS.", "tool_calls": [ { "name": "update_settings", "arguments_raw": "{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "\nApply the dark theme and disable SMS.", "tool_calls": [ { "name": "update_settings", "arguments_raw": "{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "\nApply the dark theme and disable SMS.", "tool_calls": [ { "name": "update_settings", "arguments_raw": "{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}" } ], "exception": null }, "token": { "content": "\n", "reasoning_content": "\nApply the dark theme and disable SMS.", "tool_calls": [ { "name": "update_settings", "arguments_raw": "{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}" } ], "exception": null }, "rand:1:8": { "content": "\n", "reasoning_content": "\nApply the dark theme and disable SMS.", "tool_calls": [ { "name": "update_settings", "arguments_raw": "{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "\nApply the dark theme and disable SMS.", "tool_calls": [ { "name": "update_settings", "arguments_raw": "{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "\nApply the dark theme and disable SMS.", "tool_calls": [ { "name": "update_settings", "arguments_raw": "{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "\nApply the dark theme and disable SMS.", "tool_calls": [ { "name": "update_settings", "arguments_raw": "{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}" } ], "exception": null }, "rand:5:8": { "content": "\n", "reasoning_content": "\nApply the dark theme and disable SMS.", "tool_calls": [ { "name": "update_settings", "arguments_raw": "{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}" } ], "exception": null } } } }, { "fixture_id": "glm/glm45-unicode-emoji", "family": "glm", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "reasoning_content: expected \"Search with the user's exact wording.\", got \"\\nSearch with the user's exact wording.\"" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"Search with the user's exact wording.\", got \"\\nSearch with the user's exact wording.\"" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"Search with the user's exact wording.\", got \"\\nSearch with the user's exact wording.\"" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'; reasoning_content (whitespace only): expected \"Search with the user's exact wording.\", got \"\\nSearch with the user's exact wording.\"" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'; reasoning_content (whitespace only): expected \"Search with the user's exact wording.\", got \"\\nSearch with the user's exact wording.\"" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"Search with the user's exact wording.\", got \"\\nSearch with the user's exact wording.\"" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"Search with the user's exact wording.\", got \"\\nSearch with the user's exact wording.\"" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"Search with the user's exact wording.\", got \"\\nSearch with the user's exact wording.\"" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'; reasoning_content (whitespace only): expected \"Search with the user's exact wording.\", got \"\\nSearch with the user's exact wording.\"" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "reasoning_content: nonstream \"\\nSearch with the user's exact wording.\", got \"\\nSearch with the user's exact wording.\"" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "reasoning_content: nonstream \"\\nSearch with the user's exact wording.\", got \"\\nSearch with the user's exact wording.\"" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content (whitespace only): nonstream None, got '\\n'; reasoning_content: nonstream \"\\nSearch with the user's exact wording.\", got \"\\nSearch with the user's exact wording.\"" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content (whitespace only): nonstream None, got '\\n'; reasoning_content: nonstream \"\\nSearch with the user's exact wording.\", got \"\\nSearch with the user's exact wording.\"" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "reasoning_content: nonstream \"\\nSearch with the user's exact wording.\", got \"\\nSearch with the user's exact wording.\"" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "reasoning_content: nonstream \"\\nSearch with the user's exact wording.\", got \"\\nSearch with the user's exact wording.\"" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "reasoning_content: nonstream \"\\nSearch with the user's exact wording.\", got \"\\nSearch with the user's exact wording.\"" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content (whitespace only): nonstream None, got '\\n'; reasoning_content: nonstream \"\\nSearch with the user's exact wording.\", got \"\\nSearch with the user's exact wording.\"" }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one None, got '\\n' | rand:1:8 (soft_pass): content (whitespace only): one None, got '\\n' | rand:5:8 (soft_pass): content (whitespace only): one None, got '\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "reasoning_content contains ''" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm45", "reasoning_parser": "glm45", "tool_call_detector": "Glm4MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm45", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.5", "tokenizer": { "repo": "zai-org/GLM-4.5", "revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "\nSearch with the user's exact wording.", "tool_calls": [ { "name": "search_web", "arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "\nSearch with the user's exact wording.", "tool_calls": [ { "name": "search_web", "arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "\nSearch with the user's exact wording.", "tool_calls": [ { "name": "search_web", "arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}" } ], "exception": null }, "token": { "content": "\n", "reasoning_content": "\nSearch with the user's exact wording.", "tool_calls": [ { "name": "search_web", "arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}" } ], "exception": null }, "rand:1:8": { "content": "\n", "reasoning_content": "\nSearch with the user's exact wording.", "tool_calls": [ { "name": "search_web", "arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "\nSearch with the user's exact wording.", "tool_calls": [ { "name": "search_web", "arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "\nSearch with the user's exact wording.", "tool_calls": [ { "name": "search_web", "arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "\nSearch with the user's exact wording.", "tool_calls": [ { "name": "search_web", "arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}" } ], "exception": null }, "rand:5:8": { "content": "\n", "reasoning_content": "\nSearch with the user's exact wording.", "tool_calls": [ { "name": "search_web", "arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}" } ], "exception": null } } } }, { "fixture_id": "glm/glm45-empty-arguments", "family": "glm", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "reasoning_content: expected 'Just fetch the time.', got '\\nJust fetch the time.'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Just fetch the time.', got '\\nJust fetch the time.'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Just fetch the time.', got '\\nJust fetch the time.'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'; reasoning_content (whitespace only): expected 'Just fetch the time.', got '\\nJust fetch the time.'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Just fetch the time.', got '\\nJust fetch the time.'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Just fetch the time.', got '\\nJust fetch the time.'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Just fetch the time.', got '\\nJust fetch the time.'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'; reasoning_content (whitespace only): expected 'Just fetch the time.', got '\\nJust fetch the time.'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Just fetch the time.', got '\\nJust fetch the time.'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "reasoning_content: nonstream '\\nJust fetch the time.', got '\\nJust fetch the time.'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "reasoning_content: nonstream '\\nJust fetch the time.', got '\\nJust fetch the time.'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content (whitespace only): nonstream None, got '\\n'; reasoning_content: nonstream '\\nJust fetch the time.', got '\\nJust fetch the time.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "reasoning_content: nonstream '\\nJust fetch the time.', got '\\nJust fetch the time.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "reasoning_content: nonstream '\\nJust fetch the time.', got '\\nJust fetch the time.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "reasoning_content: nonstream '\\nJust fetch the time.', got '\\nJust fetch the time.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content (whitespace only): nonstream None, got '\\n'; reasoning_content: nonstream '\\nJust fetch the time.', got '\\nJust fetch the time.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "reasoning_content: nonstream '\\nJust fetch the time.', got '\\nJust fetch the time.'" }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one None, got '\\n' | rand:4:8 (soft_pass): content (whitespace only): one None, got '\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "reasoning_content contains ''" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm45", "reasoning_parser": "glm45", "tool_call_detector": "Glm4MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm45", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.5", "tokenizer": { "repo": "zai-org/GLM-4.5", "revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "\nJust fetch the time.", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "\nJust fetch the time.", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "\nJust fetch the time.", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": "\n", "reasoning_content": "\nJust fetch the time.", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "\nJust fetch the time.", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "\nJust fetch the time.", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "\nJust fetch the time.", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:4:8": { "content": "\n", "reasoning_content": "\nJust fetch the time.", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "\nJust fetch the time.", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null } } } }, { "fixture_id": "glm/glm45-marker-in-arguments", "family": "glm", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "reasoning_content: expected 'Write the format note to the docs file.', got '\\nWrite the format note to the docs file.'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\"}'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "reasoning_content (whitespace only): expected 'Write the format note to the docs file.', got '\\nWrite the format note to the docs file.'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "reasoning_content (whitespace only): expected 'Write the format note to the docs file.', got '\\nWrite the format note to the docs file.'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}'" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected None, got '\\n; keys go in .\\n'; reasoning_content (whitespace only): expected 'Write the format note to the docs file.', got '\\nWrite the format note to the docs file.'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected None, got '; keys go in .\\n'; reasoning_content (whitespace only): expected 'Write the format note to the docs file.', got '\\nWrite the format note to the docs file.'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected None, got '; keys go in .\\n'; reasoning_content (whitespace only): expected 'Write the format note to the docs file.', got '\\nWrite the format note to the docs file.'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected None, got '\\n; keys go in .\\n'; reasoning_content (whitespace only): expected 'Write the format note to the docs file.', got '\\nWrite the format note to the docs file.'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected None, got '; keys go in .\\n'; reasoning_content (whitespace only): expected 'Write the format note to the docs file.', got '\\nWrite the format note to the docs file.'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected None, got '; keys go in .\\n'; reasoning_content (whitespace only): expected 'Write the format note to the docs file.', got '\\nWrite the format note to the docs file.'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "reasoning_content: nonstream '\\nWrite the format note to the docs file.', got '\\nWrite the format note to the docs file.'; tool_calls[0].arguments: nonstream '{\"path\": \"docs/format.md\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "reasoning_content: nonstream '\\nWrite the format note to the docs file.', got '\\nWrite the format note to the docs file.'; tool_calls[0].arguments: nonstream '{\"path\": \"docs/format.md\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream None, got '\\n; keys go in .\\n'; reasoning_content: nonstream '\\nWrite the format note to the docs file.', got '\\nWrite the format note to the docs file.'; tool_calls[0].arguments: nonstream '{\"path\": \"docs/format.md\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream None, got '; keys go in .\\n'; reasoning_content: nonstream '\\nWrite the format note to the docs file.', got '\\nWrite the format note to the docs file.'; tool_calls[0].arguments: nonstream '{\"path\": \"docs/format.md\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream None, got '; keys go in .\\n'; reasoning_content: nonstream '\\nWrite the format note to the docs file.', got '\\nWrite the format note to the docs file.'; tool_calls[0].arguments: nonstream '{\"path\": \"docs/format.md\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream None, got '\\n; keys go in .\\n'; reasoning_content: nonstream '\\nWrite the format note to the docs file.', got '\\nWrite the format note to the docs file.'; tool_calls[0].arguments: nonstream '{\"path\": \"docs/format.md\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream None, got '; keys go in .\\n'; reasoning_content: nonstream '\\nWrite the format note to the docs file.', got '\\nWrite the format note to the docs file.'; tool_calls[0].arguments: nonstream '{\"path\": \"docs/format.md\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream None, got '; keys go in .\\n'; reasoning_content: nonstream '\\nWrite the format note to the docs file.', got '\\nWrite the format note to the docs file.'; tool_calls[0].arguments: nonstream '{\"path\": \"docs/format.md\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}'" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content: one None, got '\\n; keys go in .\\n' | rand:1:8 (fail): content: one None, got '; keys go in .\\n' | rand:2:8 (fail): content: one None, got '; keys go in .\\n' | rand:3:8 (fail): content: one None, got '\\n; keys go in .\\n' | rand:4:8 (fail): content: one None, got '; keys go in .\\n' | rand:5:8 (fail): content: one None, got '; keys go in .\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "reasoning_content contains ''" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "fail", "detail": "content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "fail", "detail": "content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "fail", "detail": "content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "fail", "detail": "content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "fail", "detail": "content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "fail", "detail": "content contains ''; content contains ''" }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}': Unterminated string starting at: line 1 column 39 (char 38))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}': Unterminated string starting at: line 1 column 39 (char 38))" }, { "check": "arguments_json", "strategy": "token", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}': Unterminated string starting at: line 1 column 39 (char 38))" }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}': Unterminated string starting at: line 1 column 39 (char 38))" }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}': Unterminated string starting at: line 1 column 39 (char 38))" }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}': Unterminated string starting at: line 1 column 39 (char 38))" }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}': Unterminated string starting at: line 1 column 39 (char 38))" }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}': Unterminated string starting at: line 1 column 39 (char 38))" }, { "check": "arguments_schema", "strategy": "nonstream", "status": "fail", "detail": "[0] write_file: $: 'content' is a required property" }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm45", "reasoning_parser": "glm45", "tool_call_detector": "Glm4MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm45", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.5", "tokenizer": { "repo": "zai-org/GLM-4.5", "revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "\nWrite the format note to the docs file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"docs/format.md\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "\nWrite the format note to the docs file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "\nWrite the format note to the docs file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}" } ], "exception": null }, "token": { "content": "\n; keys go in .\n", "reasoning_content": "\nWrite the format note to the docs file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}" } ], "exception": null }, "rand:1:8": { "content": "; keys go in .\n", "reasoning_content": "\nWrite the format note to the docs file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}" } ], "exception": null }, "rand:2:8": { "content": "; keys go in .\n", "reasoning_content": "\nWrite the format note to the docs file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}" } ], "exception": null }, "rand:3:8": { "content": "\n; keys go in .\n", "reasoning_content": "\nWrite the format note to the docs file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}" } ], "exception": null }, "rand:4:8": { "content": "; keys go in .\n", "reasoning_content": "\nWrite the format note to the docs file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}" } ], "exception": null }, "rand:5:8": { "content": "; keys go in .\n", "reasoning_content": "\nWrite the format note to the docs file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}" } ], "exception": null } } } }, { "fixture_id": "glm/glm45-schema-coercion", "family": "glm", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "reasoning_content: expected 'Three second timer labelled 3.', got '\\nThree second timer labelled 3.'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Three second timer labelled 3.', got '\\nThree second timer labelled 3.'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Three second timer labelled 3.', got '\\nThree second timer labelled 3.'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'; reasoning_content (whitespace only): expected 'Three second timer labelled 3.', got '\\nThree second timer labelled 3.'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Three second timer labelled 3.', got '\\nThree second timer labelled 3.'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'; reasoning_content (whitespace only): expected 'Three second timer labelled 3.', got '\\nThree second timer labelled 3.'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Three second timer labelled 3.', got '\\nThree second timer labelled 3.'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'; reasoning_content (whitespace only): expected 'Three second timer labelled 3.', got '\\nThree second timer labelled 3.'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'; reasoning_content (whitespace only): expected 'Three second timer labelled 3.', got '\\nThree second timer labelled 3.'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "reasoning_content: nonstream '\\nThree second timer labelled 3.', got '\\nThree second timer labelled 3.'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "reasoning_content: nonstream '\\nThree second timer labelled 3.', got '\\nThree second timer labelled 3.'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content (whitespace only): nonstream None, got '\\n'; reasoning_content: nonstream '\\nThree second timer labelled 3.', got '\\nThree second timer labelled 3.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "reasoning_content: nonstream '\\nThree second timer labelled 3.', got '\\nThree second timer labelled 3.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content (whitespace only): nonstream None, got '\\n'; reasoning_content: nonstream '\\nThree second timer labelled 3.', got '\\nThree second timer labelled 3.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "reasoning_content: nonstream '\\nThree second timer labelled 3.', got '\\nThree second timer labelled 3.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content (whitespace only): nonstream None, got '\\n'; reasoning_content: nonstream '\\nThree second timer labelled 3.', got '\\nThree second timer labelled 3.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content (whitespace only): nonstream None, got '\\n'; reasoning_content: nonstream '\\nThree second timer labelled 3.', got '\\nThree second timer labelled 3.'" }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one None, got '\\n' | rand:2:8 (soft_pass): content (whitespace only): one None, got '\\n' | rand:4:8 (soft_pass): content (whitespace only): one None, got '\\n' | rand:5:8 (soft_pass): content (whitespace only): one None, got '\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "reasoning_content contains ''" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm45", "reasoning_parser": "glm45", "tool_call_detector": "Glm4MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm45", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.5", "tokenizer": { "repo": "zai-org/GLM-4.5", "revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "\nThree second timer labelled 3.", "tool_calls": [ { "name": "set_timer", "arguments_raw": "{\"seconds\": 3, \"label\": \"3\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "\nThree second timer labelled 3.", "tool_calls": [ { "name": "set_timer", "arguments_raw": "{\"seconds\": 3, \"label\": \"3\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "\nThree second timer labelled 3.", "tool_calls": [ { "name": "set_timer", "arguments_raw": "{\"seconds\": 3, \"label\": \"3\"}" } ], "exception": null }, "token": { "content": "\n", "reasoning_content": "\nThree second timer labelled 3.", "tool_calls": [ { "name": "set_timer", "arguments_raw": "{\"seconds\": 3, \"label\": \"3\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "\nThree second timer labelled 3.", "tool_calls": [ { "name": "set_timer", "arguments_raw": "{\"seconds\": 3, \"label\": \"3\"}" } ], "exception": null }, "rand:2:8": { "content": "\n", "reasoning_content": "\nThree second timer labelled 3.", "tool_calls": [ { "name": "set_timer", "arguments_raw": "{\"seconds\": 3, \"label\": \"3\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "\nThree second timer labelled 3.", "tool_calls": [ { "name": "set_timer", "arguments_raw": "{\"seconds\": 3, \"label\": \"3\"}" } ], "exception": null }, "rand:4:8": { "content": "\n", "reasoning_content": "\nThree second timer labelled 3.", "tool_calls": [ { "name": "set_timer", "arguments_raw": "{\"seconds\": 3, \"label\": \"3\"}" } ], "exception": null }, "rand:5:8": { "content": "\n", "reasoning_content": "\nThree second timer labelled 3.", "tool_calls": [ { "name": "set_timer", "arguments_raw": "{\"seconds\": 3, \"label\": \"3\"}" } ], "exception": null } } } }, { "fixture_id": "glm/glm45-thinking-disabled", "family": "glm", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one None, got '\\n' | rand:2:8 (soft_pass): content (whitespace only): one None, got '\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm45", "reasoning_parser": "glm45", "tool_call_detector": "Glm4MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm45", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.5", "tokenizer": { "repo": "zai-org/GLM-4.5", "revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 198, 151350, 151351 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius\"}" } ], "exception": null }, "token": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius\"}" } ], "exception": null }, "rand:2:8": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius\"}" } ], "exception": null } } } }, { "fixture_id": "glm/glm45-call-order-vs-tool-order", "family": "glm", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "reasoning_content: expected 'Weather, then search, then weather again.', got '\\nWeather, then search, then weather again.'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "reasoning_content (whitespace only): expected 'Weather, then search, then weather again.', got '\\nWeather, then search, then weather again.'; tool_calls: expected ['get_weather', 'search_web', 'get_weather'], got ['get_weather']" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "reasoning_content (whitespace only): expected 'Weather, then search, then weather again.', got '\\nWeather, then search, then weather again.'; tool_calls: expected ['get_weather', 'search_web', 'get_weather'], got ['get_weather']" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'; reasoning_content (whitespace only): expected 'Weather, then search, then weather again.', got '\\nWeather, then search, then weather again.'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Weather, then search, then weather again.', got '\\nWeather, then search, then weather again.'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Weather, then search, then weather again.', got '\\nWeather, then search, then weather again.'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'; reasoning_content (whitespace only): expected 'Weather, then search, then weather again.', got '\\nWeather, then search, then weather again.'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Weather, then search, then weather again.', got '\\nWeather, then search, then weather again.'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Weather, then search, then weather again.', got '\\nWeather, then search, then weather again.'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "reasoning_content: nonstream '\\nWeather, then search, then weather again.', got '\\nWeather, then search, then weather again.'; tool_calls: nonstream ['get_weather', 'search_web', 'get_weather'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "reasoning_content: nonstream '\\nWeather, then search, then weather again.', got '\\nWeather, then search, then weather again.'; tool_calls: nonstream ['get_weather', 'search_web', 'get_weather'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content (whitespace only): nonstream None, got '\\n\\n\\n'; reasoning_content: nonstream '\\nWeather, then search, then weather again.', got '\\nWeather, then search, then weather again.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "reasoning_content: nonstream '\\nWeather, then search, then weather again.', got '\\nWeather, then search, then weather again.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "reasoning_content: nonstream '\\nWeather, then search, then weather again.', got '\\nWeather, then search, then weather again.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content (whitespace only): nonstream None, got '\\n'; reasoning_content: nonstream '\\nWeather, then search, then weather again.', got '\\nWeather, then search, then weather again.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "reasoning_content: nonstream '\\nWeather, then search, then weather again.', got '\\nWeather, then search, then weather again.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "reasoning_content: nonstream '\\nWeather, then search, then weather again.', got '\\nWeather, then search, then weather again.'" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content (whitespace only): one None, got '\\n\\n\\n'; tool_calls: one ['get_weather'], got ['get_weather', 'search_web', 'get_weather'] | rand:1:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search_web', 'get_weather'] | rand:2:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search_web', 'get_weather'] | rand:3:8 (fail): content (whitespace only): one None, got '\\n'; tool_calls: one ['get_weather'], got ['get_weather', 'search_web', 'get_weather'] | rand:4:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search_web', 'get_weather'] | rand:5:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search_web', 'get_weather']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "reasoning_content contains ''" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "fail", "detail": "expected 3 calls ['get_weather', 'search_web', 'get_weather'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "special", "status": "fail", "detail": "expected 3 calls ['get_weather', 'search_web', 'get_weather'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm45", "reasoning_parser": "glm45", "tool_call_detector": "Glm4MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm45", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.5", "tokenizer": { "repo": "zai-org/GLM-4.5", "revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "\nWeather, then search, then weather again.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\"}" }, { "name": "search_web", "arguments_raw": "{\"query\": \"Oslo museums\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Bergen\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "\nWeather, then search, then weather again.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "\nWeather, then search, then weather again.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\"}" } ], "exception": null }, "token": { "content": "\n\n\n", "reasoning_content": "\nWeather, then search, then weather again.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\"}" }, { "name": "search_web", "arguments_raw": "{\"query\": \"Oslo museums\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Bergen\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "\nWeather, then search, then weather again.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\"}" }, { "name": "search_web", "arguments_raw": "{\"query\": \"Oslo museums\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Bergen\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "\nWeather, then search, then weather again.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\"}" }, { "name": "search_web", "arguments_raw": "{\"query\": \"Oslo museums\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Bergen\"}" } ], "exception": null }, "rand:3:8": { "content": "\n", "reasoning_content": "\nWeather, then search, then weather again.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\"}" }, { "name": "search_web", "arguments_raw": "{\"query\": \"Oslo museums\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Bergen\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "\nWeather, then search, then weather again.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\"}" }, { "name": "search_web", "arguments_raw": "{\"query\": \"Oslo museums\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Bergen\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "\nWeather, then search, then weather again.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\"}" }, { "name": "search_web", "arguments_raw": "{\"query\": \"Oslo museums\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Bergen\"}" } ], "exception": null } } } }, { "fixture_id": "glm/glm47-single-call", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm47", "reasoning_parser": "glm45", "tool_call_detector": "Glm47MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm47", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.7", "tokenizer": { "repo": "zai-org/GLM-4.7", "revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 151350 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "glm/glm47-parallel-same-tool", "family": "glm", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_weather'], got ['get_weather']" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_weather'], got ['get_weather']" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'get_weather'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'get_weather'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls: one ['get_weather'], got ['get_weather', 'get_weather'] | rand:1:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'get_weather'] | rand:2:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'get_weather'] | rand:3:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'get_weather'] | rand:4:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'get_weather'] | rand:5:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'get_weather']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "fail", "detail": "expected 2 calls ['get_weather', 'get_weather'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "special", "status": "fail", "detail": "expected 2 calls ['get_weather', 'get_weather'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm47", "reasoning_parser": "glm45", "tool_call_detector": "Glm47MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm47", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.7", "tokenizer": { "repo": "zai-org/GLM-4.7", "revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 151350 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "Two cities, so two independent weather lookups.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "Two cities, so two independent weather lookups.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "Two cities, so two independent weather lookups.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "token": { "content": null, "reasoning_content": "Two cities, so two independent weather lookups.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "Two cities, so two independent weather lookups.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "Two cities, so two independent weather lookups.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "Two cities, so two independent weather lookups.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "Two cities, so two independent weather lookups.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "Two cities, so two independent weather lookups.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null } } } }, { "fixture_id": "glm/glm47-content-before-call", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm47", "reasoning_parser": "glm45", "tool_call_detector": "Glm47MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm47", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.7", "tokenizer": { "repo": "zai-org/GLM-4.7", "revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 151350 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "glm/glm47-no-call", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm47", "reasoning_parser": "glm45", "tool_call_detector": "Glm47MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm47", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.7", "tokenizer": { "repo": "zai-org/GLM-4.7", "revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 151350 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "glm/glm47-nested-object", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm47", "reasoning_parser": "glm45", "tool_call_detector": "Glm47MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm47", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.7", "tokenizer": { "repo": "zai-org/GLM-4.7", "revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 151350 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "glm/glm47-unicode-emoji", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm47", "reasoning_parser": "glm45", "tool_call_detector": "Glm47MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm47", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.7", "tokenizer": { "repo": "zai-org/GLM-4.7", "revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 151350 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "glm/glm47-empty-arguments", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm47", "reasoning_parser": "glm45", "tool_call_detector": "Glm47MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm47", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.7", "tokenizer": { "repo": "zai-org/GLM-4.7", "revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 151350 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "glm/glm47-marker-in-arguments", "family": "glm", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '; keys go in .'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\"}'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}'" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected None, got '; keys go in .'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected None, got '; keys go in .'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected None, got '; keys go in .'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected None, got '; keys go in .'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected None, got '; keys go in .'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected None, got '; keys go in .'; tool_calls[0].arguments: expected '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and ; keys go in .\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '; keys go in .', got None; tool_calls[0].arguments: nonstream '{\"path\": \"docs/format.md\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '; keys go in .', got None; tool_calls[0].arguments: nonstream '{\"path\": \"docs/format.md\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '; keys go in .', got '; keys go in .'; tool_calls[0].arguments: nonstream '{\"path\": \"docs/format.md\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '; keys go in .', got '; keys go in .'; tool_calls[0].arguments: nonstream '{\"path\": \"docs/format.md\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '; keys go in .', got '; keys go in .'; tool_calls[0].arguments: nonstream '{\"path\": \"docs/format.md\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '; keys go in .', got '; keys go in .'; tool_calls[0].arguments: nonstream '{\"path\": \"docs/format.md\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '; keys go in .', got '; keys go in .'; tool_calls[0].arguments: nonstream '{\"path\": \"docs/format.md\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '; keys go in .', got '; keys go in .'; tool_calls[0].arguments: nonstream '{\"path\": \"docs/format.md\"}', got '{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}'" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content: one None, got '; keys go in .' | rand:1:8 (fail): content: one None, got '; keys go in .' | rand:2:8 (fail): content: one None, got '; keys go in .' | rand:3:8 (fail): content: one None, got '; keys go in .' | rand:4:8 (fail): content: one None, got '; keys go in .' | rand:5:8 (fail): content: one None, got '; keys go in .'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "fail", "detail": "content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "fail", "detail": "content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "fail", "detail": "content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "fail", "detail": "content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "fail", "detail": "content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "fail", "detail": "content contains ''; content contains ''" }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}': Unterminated string starting at: line 1 column 39 (char 38))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}': Unterminated string starting at: line 1 column 39 (char 38))" }, { "check": "arguments_json", "strategy": "token", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}': Unterminated string starting at: line 1 column 39 (char 38))" }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}': Unterminated string starting at: line 1 column 39 (char 38))" }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}': Unterminated string starting at: line 1 column 39 (char 38))" }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}': Unterminated string starting at: line 1 column 39 (char 38))" }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}': Unterminated string starting at: line 1 column 39 (char 38))" }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}': Unterminated string starting at: line 1 column 39 (char 38))" }, { "check": "arguments_schema", "strategy": "nonstream", "status": "fail", "detail": "[0] write_file: $: 'content' is a required property" }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm47", "reasoning_parser": "glm45", "tool_call_detector": "Glm47MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm47", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.7", "tokenizer": { "repo": "zai-org/GLM-4.7", "revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 151350 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "; keys go in .", "reasoning_content": "Write the format note to the docs file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"docs/format.md\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "Write the format note to the docs file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "Write the format note to the docs file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}" } ], "exception": null }, "token": { "content": "; keys go in .", "reasoning_content": "Write the format note to the docs file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}" } ], "exception": null }, "rand:1:8": { "content": "; keys go in .", "reasoning_content": "Write the format note to the docs file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}" } ], "exception": null }, "rand:2:8": { "content": "; keys go in .", "reasoning_content": "Write the format note to the docs file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}" } ], "exception": null }, "rand:3:8": { "content": "; keys go in .", "reasoning_content": "Write the format note to the docs file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}" } ], "exception": null }, "rand:4:8": { "content": "; keys go in .", "reasoning_content": "Write the format note to the docs file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}" } ], "exception": null }, "rand:5:8": { "content": "; keys go in .", "reasoning_content": "Write the format note to the docs file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"docs/format.md\", \"content\": \"Wrap each call in and}" } ], "exception": null } } } }, { "fixture_id": "glm/glm47-typed-values", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm47", "reasoning_parser": "glm45", "tool_call_detector": "Glm47MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm47", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.7", "tokenizer": { "repo": "zai-org/GLM-4.7", "revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 151350 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "glm/glm47-number-looking-string", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm47", "reasoning_parser": "glm45", "tool_call_detector": "Glm47MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm47", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.7", "tokenizer": { "repo": "zai-org/GLM-4.7", "revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 151350 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "glm/glm47-schema-coercion", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm47", "reasoning_parser": "glm45", "tool_call_detector": "Glm47MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm47", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.7", "tokenizer": { "repo": "zai-org/GLM-4.7", "revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 151350 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "glm/glm47-string-escapes", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm47", "reasoning_parser": "glm45", "tool_call_detector": "Glm47MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm47", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.7", "tokenizer": { "repo": "zai-org/GLM-4.7", "revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 151350 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "glm/glm47-significant-whitespace", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm47", "reasoning_parser": "glm45", "tool_call_detector": "Glm47MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm47", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.7", "tokenizer": { "repo": "zai-org/GLM-4.7", "revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 151350 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "glm/glm47-long-arguments", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm47", "reasoning_parser": "glm45", "tool_call_detector": "Glm47MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm47", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.7", "tokenizer": { "repo": "zai-org/GLM-4.7", "revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 151350 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "glm/glm47-json-looking-string", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm47", "reasoning_parser": "glm45", "tool_call_detector": "Glm47MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm47", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.7", "tokenizer": { "repo": "zai-org/GLM-4.7", "revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 151350 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "glm/glm47-thinking-disabled", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm47", "reasoning_parser": "glm45", "tool_call_detector": "Glm47MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm47", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.7", "tokenizer": { "repo": "zai-org/GLM-4.7", "revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 151351 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "glm/glm47-call-order-vs-tool-order", "family": "glm", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'search_web', 'get_weather'], got ['get_weather']" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'search_web', 'get_weather'], got ['get_weather']" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'search_web', 'get_weather'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'search_web', 'get_weather'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search_web', 'get_weather'] | rand:1:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search_web', 'get_weather'] | rand:2:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search_web', 'get_weather'] | rand:3:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search_web', 'get_weather'] | rand:4:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search_web', 'get_weather'] | rand:5:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search_web', 'get_weather']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "fail", "detail": "expected 3 calls ['get_weather', 'search_web', 'get_weather'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "special", "status": "fail", "detail": "expected 3 calls ['get_weather', 'search_web', 'get_weather'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm47", "reasoning_parser": "glm45", "tool_call_detector": "Glm47MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm47", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.7", "tokenizer": { "repo": "zai-org/GLM-4.7", "revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 151350 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "Weather, then search, then weather again.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\"}" }, { "name": "search_web", "arguments_raw": "{\"query\": \"Oslo museums\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Bergen\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "Weather, then search, then weather again.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "Weather, then search, then weather again.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\"}" } ], "exception": null }, "token": { "content": null, "reasoning_content": "Weather, then search, then weather again.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\"}" }, { "name": "search_web", "arguments_raw": "{\"query\": \"Oslo museums\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Bergen\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "Weather, then search, then weather again.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\"}" }, { "name": "search_web", "arguments_raw": "{\"query\": \"Oslo museums\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Bergen\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "Weather, then search, then weather again.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\"}" }, { "name": "search_web", "arguments_raw": "{\"query\": \"Oslo museums\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Bergen\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "Weather, then search, then weather again.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\"}" }, { "name": "search_web", "arguments_raw": "{\"query\": \"Oslo museums\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Bergen\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "Weather, then search, then weather again.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\"}" }, { "name": "search_web", "arguments_raw": "{\"query\": \"Oslo museums\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Bergen\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "Weather, then search, then weather again.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\"}" }, { "name": "search_web", "arguments_raw": "{\"query\": \"Oslo museums\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Bergen\"}" } ], "exception": null } } } }, { "fixture_id": "glm/glm47-multi-turn-second-call", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm47", "reasoning_parser": "glm45", "tool_call_detector": "Glm47MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm47", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.7", "tokenizer": { "repo": "zai-org/GLM-4.7", "revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 151350 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "glm/glm53-single-call", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm47", "reasoning_parser": "glm45", "tool_call_detector": "Glm47MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm47", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-5.3", "tokenizer": { "repo": "zai-org/GLM-5.3", "revision": "aca966e4e02791568aa6a4ced368624b3d897f42", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GlmMoeDsaForCausalLM" ], "model_type": "glm_moe_dsa" }, "hf_config_error": null, "chat_template_sha256": "3740abcea51c45830cb3ca562084ad5fb2ef53589376f73332e9886f93ade41c", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 154828, 154841 ] }, "stop": { "token": "<|endoftext|>", "id": 154820, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "glm/glm53-parallel-same-tool", "family": "glm", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_weather'], got ['get_weather']" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_weather'], got ['get_weather']" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'get_weather'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'get_weather'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls: one ['get_weather'], got ['get_weather', 'get_weather'] | rand:1:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'get_weather'] | rand:2:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'get_weather'] | rand:3:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'get_weather'] | rand:4:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'get_weather'] | rand:5:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'get_weather']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "fail", "detail": "expected 2 calls ['get_weather', 'get_weather'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "special", "status": "fail", "detail": "expected 2 calls ['get_weather', 'get_weather'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm47", "reasoning_parser": "glm45", "tool_call_detector": "Glm47MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm47", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-5.3", "tokenizer": { "repo": "zai-org/GLM-5.3", "revision": "aca966e4e02791568aa6a4ced368624b3d897f42", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GlmMoeDsaForCausalLM" ], "model_type": "glm_moe_dsa" }, "hf_config_error": null, "chat_template_sha256": "3740abcea51c45830cb3ca562084ad5fb2ef53589376f73332e9886f93ade41c", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 154828, 154841 ] }, "stop": { "token": "<|endoftext|>", "id": 154820, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "Two cities, so two independent weather lookups.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "Two cities, so two independent weather lookups.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "Two cities, so two independent weather lookups.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "token": { "content": null, "reasoning_content": "Two cities, so two independent weather lookups.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "Two cities, so two independent weather lookups.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "Two cities, so two independent weather lookups.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "Two cities, so two independent weather lookups.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "Two cities, so two independent weather lookups.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "Two cities, so two independent weather lookups.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null } } } }, { "fixture_id": "glm/glm53-no-call", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm47", "reasoning_parser": "glm45", "tool_call_detector": "Glm47MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm47", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-5.3", "tokenizer": { "repo": "zai-org/GLM-5.3", "revision": "aca966e4e02791568aa6a4ced368624b3d897f42", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GlmMoeDsaForCausalLM" ], "model_type": "glm_moe_dsa" }, "hf_config_error": null, "chat_template_sha256": "3740abcea51c45830cb3ca562084ad5fb2ef53589376f73332e9886f93ade41c", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 154828, 154841 ] }, "stop": { "token": "<|endoftext|>", "id": 154820, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "glm/glm53-schema-coercion", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm47", "reasoning_parser": "glm45", "tool_call_detector": "Glm47MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm47", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-5.3", "tokenizer": { "repo": "zai-org/GLM-5.3", "revision": "aca966e4e02791568aa6a4ced368624b3d897f42", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GlmMoeDsaForCausalLM" ], "model_type": "glm_moe_dsa" }, "hf_config_error": null, "chat_template_sha256": "3740abcea51c45830cb3ca562084ad5fb2ef53589376f73332e9886f93ade41c", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 154828, 154841 ] }, "stop": { "token": "<|endoftext|>", "id": 154820, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "glm/glm45-vllm-newline-format-content-before", "family": "glm", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected \"I'll check it.\", got None" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected \"I'll check it.\", got None" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected \"I'll check it.\", got \"I'll check it. \"" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected \"I'll check it.\", got \"I'll check it\"" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected \"I'll check it.\", got \"I'll check it\"" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream \"I'll check it.\", got None" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream \"I'll check it.\", got None" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream \"I'll check it.\", got \"I'll check it. \"" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream \"I'll check it.\", got \"I'll check it\"" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream \"I'll check it.\", got \"I'll check it\"" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content: one None, got \"I'll check it. \" | rand:1:8 (fail): content: one None, got \"I'll check it.\" | rand:2:8 (fail): content: one None, got \"I'll check it.\" | rand:3:8 (fail): content: one None, got \"I'll check it\" | rand:4:8 (fail): content: one None, got \"I'll check it\" | rand:5:8 (fail): content: one None, got \"I'll check it.\"" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm45", "reasoning_parser": "glm45", "tool_call_detector": "Glm4MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm45", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.5", "tokenizer": { "repo": "zai-org/GLM-4.5", "revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 3, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 198, 151350, 151351 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "I'll check it.", "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Dallas\", \"state\": \"TX\", \"unit\": \"fahrenheit\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Dallas\", \"state\": \"TX\", \"unit\": \"fahrenheit\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Dallas\", \"state\": \"TX\", \"unit\": \"fahrenheit\"}" } ], "exception": null }, "token": { "content": "I'll check it. ", "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Dallas\", \"state\": \"TX\", \"unit\": \"fahrenheit\"}" } ], "exception": null }, "rand:1:8": { "content": "I'll check it.", "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Dallas\", \"state\": \"TX\", \"unit\": \"fahrenheit\"}" } ], "exception": null }, "rand:2:8": { "content": "I'll check it.", "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Dallas\", \"state\": \"TX\", \"unit\": \"fahrenheit\"}" } ], "exception": null }, "rand:3:8": { "content": "I'll check it", "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Dallas\", \"state\": \"TX\", \"unit\": \"fahrenheit\"}" } ], "exception": null }, "rand:4:8": { "content": "I'll check it", "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Dallas\", \"state\": \"TX\", \"unit\": \"fahrenheit\"}" } ], "exception": null }, "rand:5:8": { "content": "I'll check it.", "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Dallas\", \"state\": \"TX\", \"unit\": \"fahrenheit\"}" } ], "exception": null } } } }, { "fixture_id": "glm/glm45-vllm-newline-format-two-calls", "family": "glm", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_current_weather', 'get_current_weather'], got ['get_current_weather']" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['get_current_weather', 'get_current_weather'], got ['get_current_weather']" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream ['get_current_weather', 'get_current_weather'], got ['get_current_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls: nonstream ['get_current_weather', 'get_current_weather'], got ['get_current_weather']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content (whitespace only): one None, got '\\n'; tool_calls: one ['get_current_weather'], got ['get_current_weather', 'get_current_weather'] | rand:1:8 (fail): tool_calls: one ['get_current_weather'], got ['get_current_weather', 'get_current_weather'] | rand:2:8 (fail): tool_calls: one ['get_current_weather'], got ['get_current_weather', 'get_current_weather'] | rand:3:8 (fail): tool_calls: one ['get_current_weather'], got ['get_current_weather', 'get_current_weather'] | rand:4:8 (fail): tool_calls: one ['get_current_weather'], got ['get_current_weather', 'get_current_weather'] | rand:5:8 (fail): tool_calls: one ['get_current_weather'], got ['get_current_weather', 'get_current_weather']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "fail", "detail": "expected 2 calls ['get_current_weather', 'get_current_weather'], got 1 ['get_current_weather']" }, { "check": "parallel_order", "strategy": "special", "status": "fail", "detail": "expected 2 calls ['get_current_weather', 'get_current_weather'], got 1 ['get_current_weather']" }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm45", "reasoning_parser": "glm45", "tool_call_detector": "Glm4MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm45", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.5", "tokenizer": { "repo": "zai-org/GLM-4.5", "revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 3, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 198, 151350, 151351 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Dallas\"}" }, { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Orlando\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Dallas\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Dallas\"}" } ], "exception": null }, "token": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Dallas\"}" }, { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Orlando\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Dallas\"}" }, { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Orlando\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Dallas\"}" }, { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Orlando\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Dallas\"}" }, { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Orlando\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Dallas\"}" }, { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Orlando\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Dallas\"}" }, { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Orlando\"}" } ], "exception": null } } } }, { "fixture_id": "glm/glm45-vllm-coerces-schema-types", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm45", "reasoning_parser": "glm45", "tool_call_detector": "Glm4MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm45", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.5", "tokenizer": { "repo": "zai-org/GLM-4.5", "revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 3, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 198, 151350, 151351 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "glm/glm45-vllm-zero-arg-newline", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm45", "reasoning_parser": "glm45", "tool_call_detector": "Glm4MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm45", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.5", "tokenizer": { "repo": "zai-org/GLM-4.5", "revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 3, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 198, 151350, 151351 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "glm/glm45-vllm-with-think-tags", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm45", "reasoning_parser": "glm45", "tool_call_detector": "Glm4MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm45", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.5", "tokenizer": { "repo": "zai-org/GLM-4.5", "revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "glm/glm45-ollama-content-after-call", "family": "glm", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected 'after tool', got None; tool_calls: expected ['test'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected 'after tool', got None; tool_calls: expected ['test'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected 'after tool', got None; tool_calls: expected ['test'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected 'after tool', got None; tool_calls: expected ['test'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected 'after tool', got None; tool_calls: expected ['test'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected 'after tool', got None; tool_calls: expected ['test'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected 'after tool', got None; tool_calls: expected ['test'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected 'after tool', got None; tool_calls: expected ['test'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected 'after tool', got None; tool_calls: expected ['test'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm45", "reasoning_parser": "glm45", "tool_call_detector": "Glm4MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm45", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.5", "tokenizer": { "repo": "zai-org/GLM-4.5", "revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "thinking", "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "thinking", "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": "thinking", "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": "thinking", "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "thinking", "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "thinking", "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "thinking", "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "thinking", "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "thinking", "tool_calls": [], "exception": null } } } }, { "fixture_id": "glm/glm47-vllm-whitespace-preserved-in-value", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm47", "reasoning_parser": "glm45", "tool_call_detector": "Glm47MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm47", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.7", "tokenizer": { "repo": "zai-org/GLM-4.7", "revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 151351 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "glm/glm47-vllm-content-before-no-space", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm47", "reasoning_parser": "glm45", "tool_call_detector": "Glm47MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm47", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.7", "tokenizer": { "repo": "zai-org/GLM-4.7", "revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 151351 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "glm/glm47-vllm-whitespace-only-content", "family": "glm", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected None, got ' \\n '" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected None, got ' \\n '" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got ' \\n '" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got ' \\n '" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got ' \\n '" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got ' \\n '" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got ' \\n '" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got ' \\n '" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got ' \\n '" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got ' \\n '" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got ' \\n '" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got ' \\n '" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got ' \\n '" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got ' \\n '" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got ' \\n '" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got ' \\n '" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm47", "reasoning_parser": "glm45", "tool_call_detector": "Glm47MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm47", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.7", "tokenizer": { "repo": "zai-org/GLM-4.7", "revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 151351 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_current_date", "arguments_raw": "{}" } ], "exception": null }, "streams": { "one": { "content": " \n ", "reasoning_content": null, "tool_calls": [ { "name": "get_current_date", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": " \n ", "reasoning_content": null, "tool_calls": [ { "name": "get_current_date", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": " \n ", "reasoning_content": null, "tool_calls": [ { "name": "get_current_date", "arguments_raw": "{}" } ], "exception": null }, "rand:1:8": { "content": " \n ", "reasoning_content": null, "tool_calls": [ { "name": "get_current_date", "arguments_raw": "{}" } ], "exception": null }, "rand:2:8": { "content": " \n ", "reasoning_content": null, "tool_calls": [ { "name": "get_current_date", "arguments_raw": "{}" } ], "exception": null }, "rand:3:8": { "content": " \n ", "reasoning_content": null, "tool_calls": [ { "name": "get_current_date", "arguments_raw": "{}" } ], "exception": null }, "rand:4:8": { "content": " \n ", "reasoning_content": null, "tool_calls": [ { "name": "get_current_date", "arguments_raw": "{}" } ], "exception": null }, "rand:5:8": { "content": " \n ", "reasoning_content": null, "tool_calls": [ { "name": "get_current_date", "arguments_raw": "{}" } ], "exception": null } } } }, { "fixture_id": "glm/glm47-vllm-args-with-newlines", "family": "glm", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm47", "reasoning_parser": "glm45", "tool_call_detector": "Glm47MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm47", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.7", "tokenizer": { "repo": "zai-org/GLM-4.7", "revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 151351 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Beijing\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Beijing\"}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Beijing\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Beijing\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Beijing\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Beijing\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Beijing\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Beijing\"}" } ], "exception": null } } } }, { "fixture_id": "glm/glm47-vllm-reasoning-without-open-tag", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm47", "reasoning_parser": "glm45", "tool_call_detector": "Glm47MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm47", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.7", "tokenizer": { "repo": "zai-org/GLM-4.7", "revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 151350 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "glm/glm47-bug-three-calls-same-tool", "family": "glm", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_weather', 'get_weather'], got ['get_weather']" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_weather', 'get_weather'], got ['get_weather']" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'get_weather', 'get_weather'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'get_weather', 'get_weather'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content (whitespace only): one None, got '\\n\\n'; tool_calls: one ['get_weather'], got ['get_weather', 'get_weather', 'get_weather'] | rand:1:8 (fail): content (whitespace only): one None, got '\\n\\n'; tool_calls: one ['get_weather'], got ['get_weather', 'get_weather', 'get_weather'] | rand:2:8 (fail): content (whitespace only): one None, got '\\n\\n'; tool_calls: one ['get_weather'], got ['get_weather', 'get_weather', 'get_weather'] | rand:3:8 (fail): content (whitespace only): one None, got '\\n\\n'; tool_calls: one ['get_weather'], got ['get_weather', 'get_weather', 'get_weather'] | rand:4:8 (fail): content (whitespace only): one None, got '\\n\\n'; tool_calls: one ['get_weather'], got ['get_weather', 'get_weather', 'get_weather'] | rand:5:8 (fail): content (whitespace only): one None, got '\\n\\n'; tool_calls: one ['get_weather'], got ['get_weather', 'get_weather', 'get_weather']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "fail", "detail": "expected 3 calls ['get_weather', 'get_weather', 'get_weather'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "special", "status": "fail", "detail": "expected 3 calls ['get_weather', 'get_weather', 'get_weather'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm47", "reasoning_parser": "glm45", "tool_call_detector": "Glm47MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm47", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.7", "tokenizer": { "repo": "zai-org/GLM-4.7", "revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 151351 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"A\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"B\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"C\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"A\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"A\"}" } ], "exception": null }, "token": { "content": "\n\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"A\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"B\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"C\"}" } ], "exception": null }, "rand:1:8": { "content": "\n\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"A\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"B\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"C\"}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"A\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"B\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"C\"}" } ], "exception": null }, "rand:3:8": { "content": "\n\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"A\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"B\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"C\"}" } ], "exception": null }, "rand:4:8": { "content": "\n\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"A\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"B\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"C\"}" } ], "exception": null }, "rand:5:8": { "content": "\n\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"A\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"B\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"C\"}" } ], "exception": null } } } }, { "fixture_id": "glm/glm47-truncated-inside-arg-value", "family": "glm", "status": "fail", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the second ; the call never closes." }, { "check": "expected_error", "strategy": "special", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the second ; the call never closes." }, { "check": "expected_error", "strategy": "token", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the second ; the call never closes." }, { "check": "expected_error", "strategy": "rand:1:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the second ; the call never closes." }, { "check": "expected_error", "strategy": "rand:2:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the second ; the call never closes." }, { "check": "expected_error", "strategy": "rand:3:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the second ; the call never closes." }, { "check": "expected_error", "strategy": "rand:4:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the second ; the call never closes." }, { "check": "expected_error", "strategy": "rand:5:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the second ; the call never closes." }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream 'get_weathercityBerlinunit', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream 'get_weathercityBerlinunit', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream 'get_weathercityBerlinunit', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream 'get_weathercityBerlinunit', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream 'get_weathercityBerlinunit', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream 'get_weathercityBerlinunit', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream 'get_weathercityBerlinunit', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream 'get_weathercityBerlinunit', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": ': Expecting value: line 1 column 28 (char 27))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": ': Expecting value: line 1 column 28 (char 27))" }, { "check": "arguments_json", "strategy": "token", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": ': Expecting value: line 1 column 28 (char 27))" }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": ': Expecting value: line 1 column 28 (char 27))" }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": ': Expecting value: line 1 column 28 (char 27))" }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": ': Expecting value: line 1 column 28 (char 27))" }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": ': Expecting value: line 1 column 28 (char 27))" }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": ': Expecting value: line 1 column 28 (char 27))" }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm47", "reasoning_parser": "glm45", "tool_call_detector": "Glm47MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm47", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.7", "tokenizer": { "repo": "zai-org/GLM-4.7", "revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 151351 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "get_weathercityBerlinunit", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": " } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": " } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": " } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": " } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": " } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": " } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": " } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": " } ], "exception": null } } } }, { "fixture_id": "glm/glm47-truncated-before-close-call", "family": "glm", "status": "fail", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit after the last ; is missing." }, { "check": "expected_error", "strategy": "special", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit after the last ; is missing." }, { "check": "expected_error", "strategy": "token", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit after the last ; is missing." }, { "check": "expected_error", "strategy": "rand:1:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit after the last ; is missing." }, { "check": "expected_error", "strategy": "rand:2:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit after the last ; is missing." }, { "check": "expected_error", "strategy": "rand:3:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit after the last ; is missing." }, { "check": "expected_error", "strategy": "rand:4:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit after the last ; is missing." }, { "check": "expected_error", "strategy": "rand:5:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit after the last ; is missing." }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream 'get_weathercityBerlinunitcelsius', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream 'get_weathercityBerlinunitcelsius', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream 'get_weathercityBerlinunitcelsius', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream 'get_weathercityBerlinunitcelsius', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream 'get_weathercityBerlinunitcelsius', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream 'get_weathercityBerlinunitcelsius', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream 'get_weathercityBerlinunitcelsius', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream 'get_weathercityBerlinunitcelsius', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": \"celsius\"': Expecting ',' delimiter: line 1 column 37 (char 36))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": \"celsius\"': Expecting ',' delimiter: line 1 column 37 (char 36))" }, { "check": "arguments_json", "strategy": "token", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": \"celsius\"': Expecting ',' delimiter: line 1 column 37 (char 36))" }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": \"celsius\"': Expecting ',' delimiter: line 1 column 37 (char 36))" }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": \"celsius\"': Expecting ',' delimiter: line 1 column 37 (char 36))" }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": \"celsius\"': Expecting ',' delimiter: line 1 column 37 (char 36))" }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": \"celsius\"': Expecting ',' delimiter: line 1 column 37 (char 36))" }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": \"celsius\"': Expecting ',' delimiter: line 1 column 37 (char 36))" }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm47", "reasoning_parser": "glm45", "tool_call_detector": "Glm47MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm47", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.7", "tokenizer": { "repo": "zai-org/GLM-4.7", "revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 151351 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "get_weathercityBerlinunitcelsius", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius\"" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius\"" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius\"" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius\"" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius\"" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius\"" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius\"" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius\"" } ], "exception": null } } } }, { "fixture_id": "glm/glm45-truncated-after-arg-key", "family": "glm", "status": "fail", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit between two arguments of an open call." }, { "check": "expected_error", "strategy": "special", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit between two arguments of an open call." }, { "check": "expected_error", "strategy": "token", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit between two arguments of an open call." }, { "check": "expected_error", "strategy": "rand:1:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit between two arguments of an open call." }, { "check": "expected_error", "strategy": "rand:2:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit between two arguments of an open call." }, { "check": "expected_error", "strategy": "rand:3:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit between two arguments of an open call." }, { "check": "expected_error", "strategy": "rand:4:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit between two arguments of an open call." }, { "check": "expected_error", "strategy": "rand:5:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit between two arguments of an open call." }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content (whitespace only): nonstream None, got '\\n'; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content (whitespace only): nonstream None, got '\\n'; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "split_invariance", "strategy": "*", "status": "soft_pass", "detail": "vs one: token (soft_pass): content (whitespace only): one None, got '\\n' | rand:2:8 (soft_pass): content (whitespace only): one None, got '\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": ': Expecting value: line 1 column 28 (char 27))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": ': Expecting value: line 1 column 28 (char 27))" }, { "check": "arguments_json", "strategy": "token", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": ': Expecting value: line 1 column 28 (char 27))" }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": ': Expecting value: line 1 column 28 (char 27))" }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": ': Expecting value: line 1 column 28 (char 27))" }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": ': Expecting value: line 1 column 28 (char 27))" }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": ': Expecting value: line 1 column 28 (char 27))" }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": ': Expecting value: line 1 column 28 (char 27))" }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm45", "reasoning_parser": "glm45", "tool_call_detector": "Glm4MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm45", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.5", "tokenizer": { "repo": "zai-org/GLM-4.5", "revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "44f815868bf02fa458dd2f741a338046f4bf45f398eb6d067766726b9d96cce3", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 198, 151350, 151351 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": " } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": " } ], "exception": null }, "token": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": " } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": " } ], "exception": null }, "rand:2:8": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": " } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": " } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": " } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": " } ], "exception": null } } } }, { "fixture_id": "glm/glm47-truncated-in-reasoning", "family": "glm", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm47", "reasoning_parser": "glm45", "tool_call_detector": "Glm47MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm47", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.7", "tokenizer": { "repo": "zai-org/GLM-4.7", "revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 151350 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "glm/glm47-missing-last-close-arg-value", "family": "glm", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Berlin\", \"unit\": \"celsius\"}', got '{\"city\": \"Berlin\"}'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Berlin\", \"unit\": \"celsius\"}', got '{\"city\": \"Berlin\", \"unit\": \"celsius}'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Berlin\", \"unit\": \"celsius\"}', got '{\"city\": \"Berlin\", \"unit\": \"celsius}'" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Berlin\", \"unit\": \"celsius\"}', got '{\"city\": \"Berlin\", \"unit\": \"celsius}'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Berlin\", \"unit\": \"celsius\"}', got '{\"city\": \"Berlin\", \"unit\": \"celsius}'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Berlin\", \"unit\": \"celsius\"}', got '{\"city\": \"Berlin\", \"unit\": \"celsius}'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Berlin\", \"unit\": \"celsius\"}', got '{\"city\": \"Berlin\", \"unit\": \"celsius}'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Berlin\", \"unit\": \"celsius\"}', got '{\"city\": \"Berlin\", \"unit\": \"celsius}'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Berlin\", \"unit\": \"celsius\"}', got '{\"city\": \"Berlin\", \"unit\": \"celsius}'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Berlin\"}', got '{\"city\": \"Berlin\", \"unit\": \"celsius}'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Berlin\"}', got '{\"city\": \"Berlin\", \"unit\": \"celsius}'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Berlin\"}', got '{\"city\": \"Berlin\", \"unit\": \"celsius}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Berlin\"}', got '{\"city\": \"Berlin\", \"unit\": \"celsius}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Berlin\"}', got '{\"city\": \"Berlin\", \"unit\": \"celsius}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Berlin\"}', got '{\"city\": \"Berlin\", \"unit\": \"celsius}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Berlin\"}', got '{\"city\": \"Berlin\", \"unit\": \"celsius}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Berlin\"}', got '{\"city\": \"Berlin\", \"unit\": \"celsius}'" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": \"celsius}': Unterminated string starting at: line 1 column 28 (char 27))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": \"celsius}': Unterminated string starting at: line 1 column 28 (char 27))" }, { "check": "arguments_json", "strategy": "token", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": \"celsius}': Unterminated string starting at: line 1 column 28 (char 27))" }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": \"celsius}': Unterminated string starting at: line 1 column 28 (char 27))" }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": \"celsius}': Unterminated string starting at: line 1 column 28 (char 27))" }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": \"celsius}': Unterminated string starting at: line 1 column 28 (char 27))" }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": \"celsius}': Unterminated string starting at: line 1 column 28 (char 27))" }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Berlin\", \"unit\": \"celsius}': Unterminated string starting at: line 1 column 28 (char 27))" }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "glm47", "reasoning_parser": "glm45", "tool_call_detector": "Glm47MoeDetector", "reasoning_detector": "Glm45Detector", "auto_detected": { "tool_call_parser": "glm47", "reasoning_parser": "glm45" }, "model": "zai-org/GLM-4.7", "tokenizer": { "repo": "zai-org/GLM-4.7", "revision": "602d01efcdd332c5238ca4bcede555defbe83eb7", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Glm4MoeForCausalLM" ], "model_type": "glm4_moe" }, "hf_config_error": null, "chat_template_sha256": "d63ad536c3c81880043e22ec7fd08db42b4d8fb7c89c7138bc562bfa25281375", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 9, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151337, 151351 ] }, "stop": { "token": "<|endoftext|>", "id": 151329, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\", \"unit\": \"celsius}" } ], "exception": null } } } }, { "fixture_id": "gpt-oss/vllm-channel-first-recipient", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] } }, { "fixture_id": "gpt-oss/vllm-channel-first-empty-arguments", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] } }, { "fixture_id": "gpt-oss/vllm-split-deltas-constrain-nospace", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '<|start|>assistant to=functions.get_weather<|channel|>commentary<|constrain|>json<|message|>{\"location\": \"Paris\"}<|call|>'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary<|constrain|>json<|message|>{\"location\": \"Paris\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary<|constrain|>json<|message|>{\"location\": \"Paris\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary<|constrain|>json<|message|>{\"location\": \"Paris\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary<|constrain|>json<|message|>{\"location\": \"Paris\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary<|constrain|>json<|message|>{\"location\": \"Paris\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary<|constrain|>json<|message|>{\"location\": \"Paris\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary<|constrain|>json<|message|>{\"location\": \"Paris\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary<|constrain|>json<|message|>{\"location\": \"Paris\"}<|call|>', got None" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '<|start|>'; content contains '<|message|>'; content contains '<|channel|>'; content contains '<|constrain|>'; content contains '<|call|>'" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] }, "observed": { "nonstream": { "content": "<|start|>assistant to=functions.get_weather<|channel|>commentary<|constrain|>json<|message|>{\"location\": \"Paris\"}<|call|>", "reasoning_content": "Thinking", "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "Thinking", "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": "Thinking", "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": "Thinking", "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "Thinking", "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "Thinking", "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "Thinking", "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "Thinking", "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "Thinking", "tool_calls": [], "exception": null } } } }, { "fixture_id": "gpt-oss/vllm-call-on-analysis-channel", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '<|start|>assistant to=functions.get_weather<|channel|>analysis<|constrain|>json<|message|>{\"location\": \"Paris\"}<|call|>'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "reasoning_content: expected 'Thinking', got 'Thinking{\"location\": \"Paris'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "reasoning_content: expected 'Thinking', got 'Thinking{\"location\":'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "reasoning_content: expected 'Thinking', got 'Thinking{\"location'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "reasoning_content: expected 'Thinking', got 'Thinking{\"location\": \"'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "reasoning_content: expected 'Thinking', got 'Thinking{\"'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "reasoning_content: expected 'Thinking', got 'Thinking{\"location\":'; tool_calls: expected ['get_weather'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>analysis<|constrain|>json<|message|>{\"location\": \"Paris\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>analysis<|constrain|>json<|message|>{\"location\": \"Paris\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>analysis<|constrain|>json<|message|>{\"location\": \"Paris\"}<|call|>', got None; reasoning_content: nonstream 'Thinking', got 'Thinking{\"location\": \"Paris'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>analysis<|constrain|>json<|message|>{\"location\": \"Paris\"}<|call|>', got None; reasoning_content: nonstream 'Thinking', got 'Thinking{\"location\":'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>analysis<|constrain|>json<|message|>{\"location\": \"Paris\"}<|call|>', got None; reasoning_content: nonstream 'Thinking', got 'Thinking{\"location'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>analysis<|constrain|>json<|message|>{\"location\": \"Paris\"}<|call|>', got None; reasoning_content: nonstream 'Thinking', got 'Thinking{\"location\": \"'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>analysis<|constrain|>json<|message|>{\"location\": \"Paris\"}<|call|>', got None; reasoning_content: nonstream 'Thinking', got 'Thinking{\"'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>analysis<|constrain|>json<|message|>{\"location\": \"Paris\"}<|call|>', got None; reasoning_content: nonstream 'Thinking', got 'Thinking{\"location\":'" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): reasoning_content: one 'Thinking', got 'Thinking{\"location\": \"Paris' | rand:1:8 (fail): reasoning_content: one 'Thinking', got 'Thinking{\"location\":' | rand:2:8 (fail): reasoning_content: one 'Thinking', got 'Thinking{\"location' | rand:3:8 (fail): reasoning_content: one 'Thinking', got 'Thinking{\"location\": \"' | rand:4:8 (fail): reasoning_content: one 'Thinking', got 'Thinking{\"' | rand:5:8 (fail): reasoning_content: one 'Thinking', got 'Thinking{\"location\":'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '<|start|>'; content contains '<|message|>'; content contains '<|channel|>'; content contains '<|constrain|>'; content contains '<|call|>'" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] }, "observed": { "nonstream": { "content": "<|start|>assistant to=functions.get_weather<|channel|>analysis<|constrain|>json<|message|>{\"location\": \"Paris\"}<|call|>", "reasoning_content": "Thinking", "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "Thinking", "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": "Thinking", "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": "Thinking{\"location\": \"Paris", "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "Thinking{\"location\":", "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "Thinking{\"location", "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "Thinking{\"location\": \"", "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "Thinking{\"", "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "Thinking{\"location\":", "tool_calls": [], "exception": null } } } }, { "fixture_id": "gpt-oss/vllm-preamble-call-end-terminated", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected 'Let me check the weather.', got 'Let me check the weather.{\"location\": \"SF\"}'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected 'Let me check the weather.', got 'Let me check the weather.{\"location\": \"SF\"}'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected 'Let me check the weather.', got 'Let me check the weather.{\"location\": \"SF\"}'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected 'Let me check the weather.', got 'Let me check the weather.{\"location\": \"SF\"}'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected 'Let me check the weather.', got 'Let me check the weather.{\"location\": \"SF\"}'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected 'Let me check the weather.', got 'Let me check the weather.{\"location\": \"SF\"}'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected 'Let me check the weather.', got 'Let me check the weather.{\"location\": \"SF\"}'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected 'Let me check the weather.', got 'Let me check the weather.{\"location\": \"SF\"}'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected 'Let me check the weather.', got 'Let me check the weather.{\"location\": \"SF\"}'; tool_calls: expected ['get_weather'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] }, "observed": { "nonstream": { "content": "Let me check the weather.{\"location\": \"SF\"}", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": "Let me check the weather.{\"location\": \"SF\"}", "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": "Let me check the weather.{\"location\": \"SF\"}", "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": "Let me check the weather.{\"location\": \"SF\"}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:1:8": { "content": "Let me check the weather.{\"location\": \"SF\"}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:2:8": { "content": "Let me check the weather.{\"location\": \"SF\"}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:3:8": { "content": "Let me check the weather.{\"location\": \"SF\"}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:4:8": { "content": "Let me check the weather.{\"location\": \"SF\"}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:5:8": { "content": "Let me check the weather.{\"location\": \"SF\"}", "reasoning_content": null, "tool_calls": [], "exception": null } } } }, { "fixture_id": "gpt-oss/vllm-sequential-calls", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '<|start|>assistant to=functions.get_weather<|channel|>commentary<|constrain|>json<|message|>{\"location\": \"Paris\"}<|call|><|start|>assistant to=functions.get...; tool_calls: expected ['get_weather', 'get_time'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_time'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_time'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_time'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_time'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_time'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_time'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_time'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_time'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary<|constrain|>json<|message|>{\"location\": \"Paris\"}<|call|><|start|>assistant to=functions.get..., got None" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary<|constrain|>json<|message|>{\"location\": \"Paris\"}<|call|><|start|>assistant to=functions.get..., got None" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary<|constrain|>json<|message|>{\"location\": \"Paris\"}<|call|><|start|>assistant to=functions.get..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary<|constrain|>json<|message|>{\"location\": \"Paris\"}<|call|><|start|>assistant to=functions.get..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary<|constrain|>json<|message|>{\"location\": \"Paris\"}<|call|><|start|>assistant to=functions.get..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary<|constrain|>json<|message|>{\"location\": \"Paris\"}<|call|><|start|>assistant to=functions.get..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary<|constrain|>json<|message|>{\"location\": \"Paris\"}<|call|><|start|>assistant to=functions.get..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary<|constrain|>json<|message|>{\"location\": \"Paris\"}<|call|><|start|>assistant to=functions.get..., got None" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '<|start|>'; content contains '<|message|>'; content contains '<|channel|>'; content contains '<|constrain|>'; content contains '<|call|>'" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "fail", "detail": "expected 2 calls ['get_weather', 'get_time'], got 0 []" }, { "check": "parallel_order", "strategy": "one", "status": "fail", "detail": "expected 2 calls ['get_weather', 'get_time'], got 0 []" }, { "check": "parallel_order", "strategy": "special", "status": "fail", "detail": "expected 2 calls ['get_weather', 'get_time'], got 0 []" }, { "check": "parallel_order", "strategy": "token", "status": "fail", "detail": "expected 2 calls ['get_weather', 'get_time'], got 0 []" }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "fail", "detail": "expected 2 calls ['get_weather', 'get_time'], got 0 []" }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "fail", "detail": "expected 2 calls ['get_weather', 'get_time'], got 0 []" }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "fail", "detail": "expected 2 calls ['get_weather', 'get_time'], got 0 []" }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "fail", "detail": "expected 2 calls ['get_weather', 'get_time'], got 0 []" }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "fail", "detail": "expected 2 calls ['get_weather', 'get_time'], got 0 []" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] }, "observed": { "nonstream": { "content": "<|start|>assistant to=functions.get_weather<|channel|>commentary<|constrain|>json<|message|>{\"location\": \"Paris\"}<|call|><|start|>assistant to=functions.get_time<|channel|>commentary<|constrain|>json<|message|>{\"timezone\": \"UTC\"}<|call|>", "reasoning_content": "Thinking", "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "Thinking", "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": "Thinking", "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": "Thinking", "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "Thinking", "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "Thinking", "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "Thinking", "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "Thinking", "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "Thinking", "tool_calls": [], "exception": null } } } }, { "fixture_id": "gpt-oss/vllm-call-then-final", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected 'Done', got '<|start|>assistant to=functions.search<|channel|>commentary<|constrain|>json<|message|>{\"query\": \"vllm\"}<|call|>Done'; tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary<|constrain|>json<|message|>{\"query\": \"vllm\"}<|call|>Done', got 'Done'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary<|constrain|>json<|message|>{\"query\": \"vllm\"}<|call|>Done', got 'Done'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary<|constrain|>json<|message|>{\"query\": \"vllm\"}<|call|>Done', got 'Done'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary<|constrain|>json<|message|>{\"query\": \"vllm\"}<|call|>Done', got 'Done'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary<|constrain|>json<|message|>{\"query\": \"vllm\"}<|call|>Done', got 'Done'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary<|constrain|>json<|message|>{\"query\": \"vllm\"}<|call|>Done', got 'Done'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary<|constrain|>json<|message|>{\"query\": \"vllm\"}<|call|>Done', got 'Done'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary<|constrain|>json<|message|>{\"query\": \"vllm\"}<|call|>Done', got 'Done'" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '<|start|>'; content contains '<|message|>'; content contains '<|channel|>'; content contains '<|constrain|>'; content contains '<|call|>'" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|return|>", "id": 200002, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "<|start|>assistant to=functions.search<|channel|>commentary<|constrain|>json<|message|>{\"query\": \"vllm\"}<|call|>Done", "reasoning_content": "Reasoning about query...", "tool_calls": [], "exception": null }, "streams": { "one": { "content": "Done", "reasoning_content": "Reasoning about query...", "tool_calls": [], "exception": null }, "special": { "content": "Done", "reasoning_content": "Reasoning about query...", "tool_calls": [], "exception": null }, "token": { "content": "Done", "reasoning_content": "Reasoning about query...", "tool_calls": [], "exception": null }, "rand:1:8": { "content": "Done", "reasoning_content": "Reasoning about query...", "tool_calls": [], "exception": null }, "rand:2:8": { "content": "Done", "reasoning_content": "Reasoning about query...", "tool_calls": [], "exception": null }, "rand:3:8": { "content": "Done", "reasoning_content": "Reasoning about query...", "tool_calls": [], "exception": null }, "rand:4:8": { "content": "Done", "reasoning_content": "Reasoning about query...", "tool_calls": [], "exception": null }, "rand:5:8": { "content": "Done", "reasoning_content": "Reasoning about query...", "tool_calls": [], "exception": null } } } }, { "fixture_id": "gpt-oss/vllm-truncated-final", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gpt-oss/vllm-interrupted-final-only", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gpt-oss/vllm-malformed-headers", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|return|>", "id": 200002, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gpt-oss/llamacpp-commentary-content-only", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected \"Hello, world!\\nWhat's up?\", got None" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected \"Hello, world!\\nWhat's up?\", got None" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected \"Hello, world!\\nWhat's up?\", got None" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected \"Hello, world!\\nWhat's up?\", got None" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected \"Hello, world!\\nWhat's up?\", got None" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected \"Hello, world!\\nWhat's up?\", got None" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected \"Hello, world!\\nWhat's up?\", got None" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected \"Hello, world!\\nWhat's up?\", got None" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected \"Hello, world!\\nWhat's up?\", got None" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|return|>", "id": 200002, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null } } } }, { "fixture_id": "gpt-oss/llamacpp-recipient-in-channel-analysis", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '<|channel|>analysis to=functions.special_function<|message|>{\"arg1\": 1}<|call|>'; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "reasoning_content: expected None, got '{\"arg1\": 1'; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "reasoning_content: expected None, got '{\"arg1\":'; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "reasoning_content: expected None, got '{\"arg1\": 1'; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "reasoning_content: expected None, got '{\"arg1\": '; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "reasoning_content: expected None, got '{\"arg1'; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "reasoning_content: expected None, got '{\"arg1\":'; tool_calls: expected ['special_function'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '<|channel|>analysis to=functions.special_function<|message|>{\"arg1\": 1}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '<|channel|>analysis to=functions.special_function<|message|>{\"arg1\": 1}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '<|channel|>analysis to=functions.special_function<|message|>{\"arg1\": 1}<|call|>', got None; reasoning_content: nonstream None, got '{\"arg1\": 1'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '<|channel|>analysis to=functions.special_function<|message|>{\"arg1\": 1}<|call|>', got None; reasoning_content: nonstream None, got '{\"arg1\":'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '<|channel|>analysis to=functions.special_function<|message|>{\"arg1\": 1}<|call|>', got None; reasoning_content: nonstream None, got '{\"arg1\": 1'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '<|channel|>analysis to=functions.special_function<|message|>{\"arg1\": 1}<|call|>', got None; reasoning_content: nonstream None, got '{\"arg1\": '" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '<|channel|>analysis to=functions.special_function<|message|>{\"arg1\": 1}<|call|>', got None; reasoning_content: nonstream None, got '{\"arg1'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '<|channel|>analysis to=functions.special_function<|message|>{\"arg1\": 1}<|call|>', got None; reasoning_content: nonstream None, got '{\"arg1\":'" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): reasoning_content: one None, got '{\"arg1\": 1' | rand:1:8 (fail): reasoning_content: one None, got '{\"arg1\":' | rand:2:8 (fail): reasoning_content: one None, got '{\"arg1\": 1' | rand:3:8 (fail): reasoning_content: one None, got '{\"arg1\": ' | rand:4:8 (fail): reasoning_content: one None, got '{\"arg1' | rand:5:8 (fail): reasoning_content: one None, got '{\"arg1\":'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '<|message|>'; content contains '<|channel|>'; content contains '<|call|>'" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] }, "observed": { "nonstream": { "content": "<|channel|>analysis to=functions.special_function<|message|>{\"arg1\": 1}<|call|>", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": "{\"arg1\": 1", "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "{\"arg1\":", "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "{\"arg1\": 1", "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "{\"arg1\": ", "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "{\"arg1", "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "{\"arg1\":", "tool_calls": [], "exception": null } } } }, { "fixture_id": "gpt-oss/llamacpp-recipient-in-channel-commentary", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '<|channel|>commentary to=functions.special_function<|message|>{\"arg1\": 1}<|call|>'; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: expected ['special_function'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '<|channel|>commentary to=functions.special_function<|message|>{\"arg1\": 1}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '<|channel|>commentary to=functions.special_function<|message|>{\"arg1\": 1}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '<|channel|>commentary to=functions.special_function<|message|>{\"arg1\": 1}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '<|channel|>commentary to=functions.special_function<|message|>{\"arg1\": 1}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '<|channel|>commentary to=functions.special_function<|message|>{\"arg1\": 1}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '<|channel|>commentary to=functions.special_function<|message|>{\"arg1\": 1}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '<|channel|>commentary to=functions.special_function<|message|>{\"arg1\": 1}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '<|channel|>commentary to=functions.special_function<|message|>{\"arg1\": 1}<|call|>', got None" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '<|message|>'; content contains '<|channel|>'; content contains '<|call|>'" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] }, "observed": { "nonstream": { "content": "<|channel|>commentary to=functions.special_function<|message|>{\"arg1\": 1}<|call|>", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null } } } }, { "fixture_id": "gpt-oss/llamacpp-recipient-in-role-constrain", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got ' to=functions.special_function<|channel|>analysis <|constrain|>json<|message|>{\"arg1\": 1}<|call|>'; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected None, got ' to=functions.special_function'; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected None, got ' to=functions.special_function to=functions.special_function'; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected None, got ' to=functions.special_function to=functions.special_function'; reasoning_content: expected None, got '{\"arg1\": 1'; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected None, got ' to=functions.special_function to=functions.special_function'; reasoning_content: expected None, got '{\"'; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected None, got ' to=functions.special_function to=functions.special_function'; reasoning_content: expected None, got '{\"arg1'; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected None, got ' to=functions.special_function to=functions.special_function'; reasoning_content: expected None, got '{\"arg'; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected None, got ' to=functions.special_function to=functions.special_function'; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected None, got ' to=functions.special_function to=functions.special_function'; reasoning_content: expected None, got '{\"'; tool_calls: expected ['special_function'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream ' to=functions.special_function<|channel|>analysis <|constrain|>json<|message|>{\"arg1\": 1}<|call|>', got ' to=functions.special_function'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream ' to=functions.special_function<|channel|>analysis <|constrain|>json<|message|>{\"arg1\": 1}<|call|>', got ' to=functions.special_function to=functions.special_function'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream ' to=functions.special_function<|channel|>analysis <|constrain|>json<|message|>{\"arg1\": 1}<|call|>', got ' to=functions.special_function to=functions.special_function'; reasoning_content: nonstream None, got '{\"arg1\": 1'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream ' to=functions.special_function<|channel|>analysis <|constrain|>json<|message|>{\"arg1\": 1}<|call|>', got ' to=functions.special_function to=functions.special_function'; reasoning_content: nonstream None, got '{\"'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream ' to=functions.special_function<|channel|>analysis <|constrain|>json<|message|>{\"arg1\": 1}<|call|>', got ' to=functions.special_function to=functions.special_function'; reasoning_content: nonstream None, got '{\"arg1'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream ' to=functions.special_function<|channel|>analysis <|constrain|>json<|message|>{\"arg1\": 1}<|call|>', got ' to=functions.special_function to=functions.special_function'; reasoning_content: nonstream None, got '{\"arg'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream ' to=functions.special_function<|channel|>analysis <|constrain|>json<|message|>{\"arg1\": 1}<|call|>', got ' to=functions.special_function to=functions.special_function'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream ' to=functions.special_function<|channel|>analysis <|constrain|>json<|message|>{\"arg1\": 1}<|call|>', got ' to=functions.special_function to=functions.special_function'; reasoning_content: nonstream None, got '{\"'" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: special (fail): content: one ' to=functions.special_function', got ' to=functions.special_function to=functions.special_function' | token (fail): content: one ' to=functions.special_function', got ' to=functions.special_function to=functions.special_function'; reasoning_content: one None, got '{\"arg1\": 1' | rand:1:8 (fail): content: one ' to=functions.special_function', got ' to=functions.special_function to=functions.special_function'; reasoning_content: one None, got '{\"' | rand:2:8 (fail): content: one ' to=functions.special_function', got ' to=functions.special_function to=functions.special_function'; reasoning_content: one None, got '{\"arg1' | rand:3:8 (fail): content: one ' to=functions.special_function', got ' to=functions.special_function to=functions.special_function'; reasoning_content: one None, got '{\"arg' | rand:4:8 (fail): content: one ' to=functions.special_function', got ' to=functions.special_function to=functions.special_function' | rand:5:8 (fail): content: one ' to=functions.special_function', got ' to=functions.special_function to=functions.special_function'; reasoning_content: one None, got '{\"'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '<|message|>'; content contains '<|channel|>'; content contains '<|constrain|>'; content contains '<|call|>'" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] }, "observed": { "nonstream": { "content": " to=functions.special_function<|channel|>analysis <|constrain|>json<|message|>{\"arg1\": 1}<|call|>", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": " to=functions.special_function", "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": " to=functions.special_function to=functions.special_function", "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": " to=functions.special_function to=functions.special_function", "reasoning_content": "{\"arg1\": 1", "tool_calls": [], "exception": null }, "rand:1:8": { "content": " to=functions.special_function to=functions.special_function", "reasoning_content": "{\"", "tool_calls": [], "exception": null }, "rand:2:8": { "content": " to=functions.special_function to=functions.special_function", "reasoning_content": "{\"arg1", "tool_calls": [], "exception": null }, "rand:3:8": { "content": " to=functions.special_function to=functions.special_function", "reasoning_content": "{\"arg", "tool_calls": [], "exception": null }, "rand:4:8": { "content": " to=functions.special_function to=functions.special_function", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:5:8": { "content": " to=functions.special_function to=functions.special_function", "reasoning_content": "{\"", "tool_calls": [], "exception": null } } } }, { "fixture_id": "gpt-oss/llamacpp-edit-code-arguments", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] } }, { "fixture_id": "gpt-oss/llamacpp-builtin-python-recipient", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] } }, { "fixture_id": "gpt-oss/llamacpp-stray-commentary-header", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|return|>", "id": 200002, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gpt-oss/harmony-test-channel-first-constrain-adjacent", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '<|channel|>commentary to=functions.get_weather<|constrain|>json<|message|>{\"latitude\":48.8566,\"longitude\":2.3522}<|call|>'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '<|channel|>commentary to=functions.get_weather<|constrain|>json<|message|>{\"latitude\":48.8566,\"longitude\":2.3522}<|call|>', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '<|channel|>commentary to=functions.get_weather<|constrain|>json<|message|>{\"latitude\":48.8566,\"longitude\":2.3522}<|call|>', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '<|channel|>commentary to=functions.get_weather<|constrain|>json<|message|>{\"latitude\":48.8566,\"longitude\":2.3522}<|call|>', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '<|channel|>commentary to=functions.get_weather<|constrain|>json<|message|>{\"latitude\":48.8566,\"longitude\":2.3522}<|call|>', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '<|channel|>commentary to=functions.get_weather<|constrain|>json<|message|>{\"latitude\":48.8566,\"longitude\":2.3522}<|call|>', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '<|channel|>commentary to=functions.get_weather<|constrain|>json<|message|>{\"latitude\":48.8566,\"longitude\":2.3522}<|call|>', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '<|channel|>commentary to=functions.get_weather<|constrain|>json<|message|>{\"latitude\":48.8566,\"longitude\":2.3522}<|call|>', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '<|channel|>commentary to=functions.get_weather<|constrain|>json<|message|>{\"latitude\":48.8566,\"longitude\":2.3522}<|call|>', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '<|message|>'; content contains '<|channel|>'; content contains '<|constrain|>'; content contains '<|call|>'" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] }, "observed": { "nonstream": { "content": "<|channel|>commentary to=functions.get_weather<|constrain|>json<|message|>{\"latitude\":48.8566,\"longitude\":2.3522}<|call|>", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"latitude\": 48.8566, \"longitude\": 2.3522}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"latitude\": 48.8566, \"longitude\": 2.3522}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"latitude\": 48.8566, \"longitude\": 2.3522}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"latitude\": 48.8566, \"longitude\": 2.3522}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"latitude\": 48.8566, \"longitude\": 2.3522}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"latitude\": 48.8566, \"longitude\": 2.3522}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"latitude\": 48.8566, \"longitude\": 2.3522}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"latitude\": 48.8566, \"longitude\": 2.3522}" } ], "exception": null } } } }, { "fixture_id": "gpt-oss/harmony-test-role-recipient-constrain-adjacent", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got ' to=functions.get_weather<|channel|>commentary<|constrain|>json<|message|>{\"location\": \"Tokyo\"}<|call|>'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected None, got ' to=functions.get_weather'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected None, got ' to=functions.get_weather to=functions.get_weather'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected None, got ' to=functions.get_weather to=functions.get_weather'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected None, got ' to=functions.get_weather to=functions.get_weather'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected None, got ' to=functions.get_weather to=functions.get_weather'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected None, got ' to=functions.get_weather to=functions.get_weather'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected None, got ' to=functions.get_weather to=functions.get_weather'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected None, got ' to=functions.get_weather to=functions.get_weather'; tool_calls: expected ['get_weather'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream ' to=functions.get_weather<|channel|>commentary<|constrain|>json<|message|>{\"location\": \"Tokyo\"}<|call|>', got ' to=functions.get_weather'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream ' to=functions.get_weather<|channel|>commentary<|constrain|>json<|message|>{\"location\": \"Tokyo\"}<|call|>', got ' to=functions.get_weather to=functions.get_weather'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream ' to=functions.get_weather<|channel|>commentary<|constrain|>json<|message|>{\"location\": \"Tokyo\"}<|call|>', got ' to=functions.get_weather to=functions.get_weather'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream ' to=functions.get_weather<|channel|>commentary<|constrain|>json<|message|>{\"location\": \"Tokyo\"}<|call|>', got ' to=functions.get_weather to=functions.get_weather'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream ' to=functions.get_weather<|channel|>commentary<|constrain|>json<|message|>{\"location\": \"Tokyo\"}<|call|>', got ' to=functions.get_weather to=functions.get_weather'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream ' to=functions.get_weather<|channel|>commentary<|constrain|>json<|message|>{\"location\": \"Tokyo\"}<|call|>', got ' to=functions.get_weather to=functions.get_weather'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream ' to=functions.get_weather<|channel|>commentary<|constrain|>json<|message|>{\"location\": \"Tokyo\"}<|call|>', got ' to=functions.get_weather to=functions.get_weather'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream ' to=functions.get_weather<|channel|>commentary<|constrain|>json<|message|>{\"location\": \"Tokyo\"}<|call|>', got ' to=functions.get_weather to=functions.get_weather'" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: special (fail): content: one ' to=functions.get_weather', got ' to=functions.get_weather to=functions.get_weather' | token (fail): content: one ' to=functions.get_weather', got ' to=functions.get_weather to=functions.get_weather' | rand:1:8 (fail): content: one ' to=functions.get_weather', got ' to=functions.get_weather to=functions.get_weather' | rand:2:8 (fail): content: one ' to=functions.get_weather', got ' to=functions.get_weather to=functions.get_weather' | rand:3:8 (fail): content: one ' to=functions.get_weather', got ' to=functions.get_weather to=functions.get_weather' | rand:4:8 (fail): content: one ' to=functions.get_weather', got ' to=functions.get_weather to=functions.get_weather' | rand:5:8 (fail): content: one ' to=functions.get_weather', got ' to=functions.get_weather to=functions.get_weather'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '<|message|>'; content contains '<|channel|>'; content contains '<|constrain|>'; content contains '<|call|>'" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] }, "observed": { "nonstream": { "content": " to=functions.get_weather<|channel|>commentary<|constrain|>json<|message|>{\"location\": \"Tokyo\"}<|call|>", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": " to=functions.get_weather", "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": " to=functions.get_weather to=functions.get_weather", "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": " to=functions.get_weather to=functions.get_weather", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:1:8": { "content": " to=functions.get_weather to=functions.get_weather", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:2:8": { "content": " to=functions.get_weather to=functions.get_weather", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:3:8": { "content": " to=functions.get_weather to=functions.get_weather", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:4:8": { "content": " to=functions.get_weather to=functions.get_weather", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:5:8": { "content": " to=functions.get_weather to=functions.get_weather", "reasoning_content": null, "tool_calls": [], "exception": null } } } }, { "fixture_id": "gpt-oss/harmony-spec-receiving-tool-calls", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] } }, { "fixture_id": "gpt-oss/harmony-spec-preamble-action-plan", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..., got '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..." }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..., got '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..." }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..., got '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..." }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..., got '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..." }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..., got '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..." }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..., got '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..." }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..., got '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..." }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..., got '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..." }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..., got '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..." }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..., got '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..." }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..., got '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..." }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..., got '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..." }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..., got '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..." }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..., got '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..." }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: special (fail): content: one '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..., got '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by... | token (fail): content: one '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..., got '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by... | rand:1:8 (fail): content: one '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..., got '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by... | rand:2:8 (fail): content: one '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..., got '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by... | rand:3:8 (fail): content: one '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..., got '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by... | rand:4:8 (fail): content: one '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..., got '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by... | rand:5:8 (fail): content: one '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..., got '**Action plan**:\\n1. Generate an HTML file\\n2. Generate a JavaScript for the Node.js server\\n3. Start the server\\n---\\nWill start executing the plan step by..." }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] }, "observed": { "nonstream": { "content": "**Action plan**:\n1. Generate an HTML file\n2. Generate a JavaScript for the Node.js server\n3. Start the server\n---\nWill start executing the plan step by step", "reasoning_content": "{long chain of thought}", "tool_calls": [ { "name": "generate_file", "arguments_raw": "{\"template\": \"basic_html\", \"path\": \"index.html\"}" } ], "exception": null }, "streams": { "one": { "content": "**Action plan**:\n1. Generate an HTML file\n2. Generate a JavaScript for the Node.js server\n3. Start the server\n---\nWill start executing the plan step by step", "reasoning_content": "{long chain of thought}", "tool_calls": [ { "name": "generate_file", "arguments_raw": "{\"template\": \"basic_html\", \"path\": \"index.html\"}" } ], "exception": null }, "special": { "content": "**Action plan**:\n1. Generate an HTML file\n2. Generate a JavaScript for the Node.js server\n3. Start the server\n---\nWill start executing the plan step by step**Action plan**:\n1. Generate an HTML file\n2. Generate a JavaScript for the Node.js server\n3. Start the server\n---\nWill start executing the plan step by step", "reasoning_content": "{long chain of thought}", "tool_calls": [ { "name": "generate_file", "arguments_raw": "{\"template\": \"basic_html\", \"path\": \"index.html\"}" } ], "exception": null }, "token": { "content": "**Action plan**:\n1. Generate an HTML file\n2. Generate a JavaScript for the Node.js server\n3. Start the server\n---\nWill start executing the plan step by step**Action plan**:\n1. Generate an HTML file\n2. Generate a JavaScript for the Node.js server\n3. Start the server\n---\nWill start executing the plan step by step", "reasoning_content": "{long chain of thought}", "tool_calls": [ { "name": "generate_file", "arguments_raw": "{\"template\": \"basic_html\", \"path\": \"index.html\"}" } ], "exception": null }, "rand:1:8": { "content": "**Action plan**:\n1. Generate an HTML file\n2. Generate a JavaScript for the Node.js server\n3. Start the server\n---\nWill start executing the plan step by step**Action plan**:\n1. Generate an HTML file\n2. Generate a JavaScript for the Node.js server\n3. Start the server\n---\nWill start executing the plan step by step", "reasoning_content": "{long chain of thought}", "tool_calls": [ { "name": "generate_file", "arguments_raw": "{\"template\": \"basic_html\", \"path\": \"index.html\"}" } ], "exception": null }, "rand:2:8": { "content": "**Action plan**:\n1. Generate an HTML file\n2. Generate a JavaScript for the Node.js server\n3. Start the server\n---\nWill start executing the plan step by step**Action plan**:\n1. Generate an HTML file\n2. Generate a JavaScript for the Node.js server\n3. Start the server\n---\nWill start executing the plan step by step", "reasoning_content": "{long chain of thought}", "tool_calls": [ { "name": "generate_file", "arguments_raw": "{\"template\": \"basic_html\", \"path\": \"index.html\"}" } ], "exception": null }, "rand:3:8": { "content": "**Action plan**:\n1. Generate an HTML file\n2. Generate a JavaScript for the Node.js server\n3. Start the server\n---\nWill start executing the plan step by step**Action plan**:\n1. Generate an HTML file\n2. Generate a JavaScript for the Node.js server\n3. Start the server\n---\nWill start executing the plan step by step", "reasoning_content": "{long chain of thought}", "tool_calls": [ { "name": "generate_file", "arguments_raw": "{\"template\": \"basic_html\", \"path\": \"index.html\"}" } ], "exception": null }, "rand:4:8": { "content": "**Action plan**:\n1. Generate an HTML file\n2. Generate a JavaScript for the Node.js server\n3. Start the server\n---\nWill start executing the plan step by step**Action plan**:\n1. Generate an HTML file\n2. Generate a JavaScript for the Node.js server\n3. Start the server\n---\nWill start executing the plan step by step", "reasoning_content": "{long chain of thought}", "tool_calls": [ { "name": "generate_file", "arguments_raw": "{\"template\": \"basic_html\", \"path\": \"index.html\"}" } ], "exception": null }, "rand:5:8": { "content": "**Action plan**:\n1. Generate an HTML file\n2. Generate a JavaScript for the Node.js server\n3. Start the server\n---\nWill start executing the plan step by step**Action plan**:\n1. Generate an HTML file\n2. Generate a JavaScript for the Node.js server\n3. Start the server\n---\nWill start executing the plan step by step", "reasoning_content": "{long chain of thought}", "tool_calls": [ { "name": "generate_file", "arguments_raw": "{\"template\": \"basic_html\", \"path\": \"index.html\"}" } ], "exception": null } } } }, { "fixture_id": "gpt-oss/bug-unknown-channel-comment", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|return|>", "id": 200002, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gpt-oss/bug-garbled-channel-question-marks", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gpt-oss/bug-garbled-channel-commentary-question", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gpt-oss/harmony-single-call", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\": \"Paris\", \"unit\": \"c\"}<|call|>'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\": \"Paris\", \"unit\": \"c\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\": \"Paris\", \"unit\": \"c\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\": \"Paris\", \"unit\": \"c\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\": \"Paris\", \"unit\": \"c\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\": \"Paris\", \"unit\": \"c\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\": \"Paris\", \"unit\": \"c\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\": \"Paris\", \"unit\": \"c\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\": \"Paris\", \"unit\": \"c\"}<|call|>', got None" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '<|start|>'; content contains '<|message|>'; content contains '<|channel|>'; content contains '<|constrain|>'; content contains '<|call|>'" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 5, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] }, "observed": { "nonstream": { "content": "<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\": \"Paris\", \"unit\": \"c\"}<|call|>", "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null } } } }, { "fixture_id": "gpt-oss/harmony-single-call-no-reasoning", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got ' to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\": \"Paris\"}<|call|>'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected None, got ' to=functions.get_weather'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected None, got ' to=functions.get_weather to=functions.get_weather'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected None, got ' to=functions.get_weather to=functions.get_weather'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected None, got ' to=functions.get_weather to=functions.get_weather'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected None, got ' to=functions.get_weather to=functions.get_weather'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected None, got ' to=functions.get_weather to=functions.get_weather'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected None, got ' to=functions.get_weather to=functions.get_weather'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected None, got ' to=functions.get_weather to=functions.get_weather'; tool_calls: expected ['get_weather'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream ' to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\": \"Paris\"}<|call|>', got ' to=functions.get_weather'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream ' to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\": \"Paris\"}<|call|>', got ' to=functions.get_weather to=functions.get_weather'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream ' to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\": \"Paris\"}<|call|>', got ' to=functions.get_weather to=functions.get_weather'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream ' to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\": \"Paris\"}<|call|>', got ' to=functions.get_weather to=functions.get_weather'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream ' to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\": \"Paris\"}<|call|>', got ' to=functions.get_weather to=functions.get_weather'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream ' to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\": \"Paris\"}<|call|>', got ' to=functions.get_weather to=functions.get_weather'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream ' to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\": \"Paris\"}<|call|>', got ' to=functions.get_weather to=functions.get_weather'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream ' to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\": \"Paris\"}<|call|>', got ' to=functions.get_weather to=functions.get_weather'" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: special (fail): content: one ' to=functions.get_weather', got ' to=functions.get_weather to=functions.get_weather' | token (fail): content: one ' to=functions.get_weather', got ' to=functions.get_weather to=functions.get_weather' | rand:1:8 (fail): content: one ' to=functions.get_weather', got ' to=functions.get_weather to=functions.get_weather' | rand:2:8 (fail): content: one ' to=functions.get_weather', got ' to=functions.get_weather to=functions.get_weather' | rand:3:8 (fail): content: one ' to=functions.get_weather', got ' to=functions.get_weather to=functions.get_weather' | rand:4:8 (fail): content: one ' to=functions.get_weather', got ' to=functions.get_weather to=functions.get_weather' | rand:5:8 (fail): content: one ' to=functions.get_weather', got ' to=functions.get_weather to=functions.get_weather'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '<|message|>'; content contains '<|channel|>'; content contains '<|constrain|>'; content contains '<|call|>'" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 5, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] }, "observed": { "nonstream": { "content": " to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\": \"Paris\"}<|call|>", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": " to=functions.get_weather", "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": " to=functions.get_weather to=functions.get_weather", "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": " to=functions.get_weather to=functions.get_weather", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:1:8": { "content": " to=functions.get_weather to=functions.get_weather", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:2:8": { "content": " to=functions.get_weather to=functions.get_weather", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:3:8": { "content": " to=functions.get_weather to=functions.get_weather", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:4:8": { "content": " to=functions.get_weather to=functions.get_weather", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:5:8": { "content": " to=functions.get_weather to=functions.get_weather", "reasoning_content": null, "tool_calls": [], "exception": null } } } }, { "fixture_id": "gpt-oss/harmony-compact-json", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\":\"San Francisco\"}<|call|>'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\":\"San Francisco\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\":\"San Francisco\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\":\"San Francisco\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\":\"San Francisco\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\":\"San Francisco\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\":\"San Francisco\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\":\"San Francisco\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\":\"San Francisco\"}<|call|>', got None" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '<|start|>'; content contains '<|message|>'; content contains '<|channel|>'; content contains '<|constrain|>'; content contains '<|call|>'" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 5, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] }, "observed": { "nonstream": { "content": "<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\":\"San Francisco\"}<|call|>", "reasoning_content": "Need to use function get_weather.", "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "Need to use function get_weather.", "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": "Need to use function get_weather.", "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": "Need to use function get_weather.", "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "Need to use function get_weather.", "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "Need to use function get_weather.", "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "Need to use function get_weather.", "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "Need to use function get_weather.", "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "Need to use function get_weather.", "tool_calls": [], "exception": null } } } }, { "fixture_id": "gpt-oss/harmony-unicode-emoji", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"東京の天気 ☀️🌧️ — «prévisions» für Zürich\", \"filters\": {\"lang\"...; tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"東京の天気 ☀️🌧️ — «prévisions» für Zürich\", \"filters\": {\"lang\"..., got None" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"東京の天気 ☀️🌧️ — «prévisions» für Zürich\", \"filters\": {\"lang\"..., got None" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"東京の天気 ☀️🌧️ — «prévisions» für Zürich\", \"filters\": {\"lang\"..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"東京の天気 ☀️🌧️ — «prévisions» für Zürich\", \"filters\": {\"lang\"..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"東京の天気 ☀️🌧️ — «prévisions» für Zürich\", \"filters\": {\"lang\"..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"東京の天気 ☀️🌧️ — «prévisions» für Zürich\", \"filters\": {\"lang\"..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"東京の天気 ☀️🌧️ — «prévisions» für Zürich\", \"filters\": {\"lang\"..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"東京の天気 ☀️🌧️ — «prévisions» für Zürich\", \"filters\": {\"lang\"..., got None" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '<|start|>'; content contains '<|message|>'; content contains '<|channel|>'; content contains '<|constrain|>'; content contains '<|call|>'" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 5, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] }, "observed": { "nonstream": { "content": "<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"東京の天気 ☀️🌧️ — «prévisions» für Zürich\", \"filters\": {\"lang\": \"ja\"}}<|call|>", "reasoning_content": "用户想知道东京的天气。Call the tool 🌦️.", "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "用户想知道东京的天气。Call the tool 🌦️.", "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": "用户想知道东京的天气。Call the tool 🌦️.", "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": "用户想知道东京的天气。Call the tool 🌦️.", "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "用户想知道东京的天气。Call the tool 🌦️.", "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "用户想知道东京的天气。Call the tool 🌦️.", "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "用户想知道东京的天气。Call the tool 🌦️.", "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "用户想知道东京的天气。Call the tool 🌦️.", "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "用户想知道东京的天气。Call the tool 🌦️.", "tool_calls": [], "exception": null } } } }, { "fixture_id": "gpt-oss/harmony-nested-json", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"range\"...; tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"range\"..., got None" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"range\"..., got None" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"range\"..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"range\"..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"range\"..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"range\"..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"range\"..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"range\"..., got None" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '<|start|>'; content contains '<|message|>'; content contains '<|channel|>'; content contains '<|constrain|>'; content contains '<|call|>'" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 5, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] }, "observed": { "nonstream": { "content": "<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"range\": {\"from\": 1, \"to\": 3, \"open\": null}, \"sort\": [{\"field\": \"rating\", \"desc\": true}], \"exact\": false}}<|call|>", "reasoning_content": "Search with structured filters.", "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "Search with structured filters.", "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": "Search with structured filters.", "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": "Search with structured filters.", "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "Search with structured filters.", "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "Search with structured filters.", "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "Search with structured filters.", "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "Search with structured filters.", "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "Search with structured filters.", "tool_calls": [], "exception": null } } } }, { "fixture_id": "gpt-oss/harmony-empty-arguments", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '<|start|>assistant to=functions.get_time<|channel|>commentary <|constrain|>json<|message|>{}<|call|>'; tool_calls: expected ['get_time'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_time'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['get_time'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls: expected ['get_time'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: expected ['get_time'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: expected ['get_time'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: expected ['get_time'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: expected ['get_time'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: expected ['get_time'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_time<|channel|>commentary <|constrain|>json<|message|>{}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_time<|channel|>commentary <|constrain|>json<|message|>{}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_time<|channel|>commentary <|constrain|>json<|message|>{}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_time<|channel|>commentary <|constrain|>json<|message|>{}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_time<|channel|>commentary <|constrain|>json<|message|>{}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_time<|channel|>commentary <|constrain|>json<|message|>{}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_time<|channel|>commentary <|constrain|>json<|message|>{}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_time<|channel|>commentary <|constrain|>json<|message|>{}<|call|>', got None" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '<|start|>'; content contains '<|message|>'; content contains '<|channel|>'; content contains '<|constrain|>'; content contains '<|call|>'" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 5, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] }, "observed": { "nonstream": { "content": "<|start|>assistant to=functions.get_time<|channel|>commentary <|constrain|>json<|message|>{}<|call|>", "reasoning_content": "Get the time.", "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "Get the time.", "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": "Get the time.", "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": "Get the time.", "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "Get the time.", "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "Get the time.", "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "Get the time.", "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "Get the time.", "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "Get the time.", "tool_calls": [], "exception": null } } } }, { "fixture_id": "gpt-oss/harmony-marker-in-arguments", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '<|start|>assistant to=functions.write_file<|channel|>commentary <|constrain|>json<|message|>{\"path\": \"harmony.md\", \"content\": \"A call ends with <|call|>; a ...; tool_calls: expected ['write_file'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected None, got '; a turn with .\\\\nHeader: {}\"}'; tool_calls: expected ['write_file'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected None, got '; a turn with .\\\\nHeader: {}\"}'; tool_calls: expected ['write_file'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected None, got '; a turn with .\\\\nHeader: {}\"}'; tool_calls: expected ['write_file'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected None, got '; a turn with .\\\\nHeader: {}\"}'; tool_calls: expected ['write_file'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected None, got '; a turn with .\\\\nHeader: {}\"}'; tool_calls: expected ['write_file'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected None, got '; a turn with .\\\\nHeader: {}\"}'; tool_calls: expected ['write_file'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected None, got '; a turn with .\\\\nHeader: {}\"}'; tool_calls: expected ['write_file'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected None, got '; a turn with .\\\\nHeader: {}\"}'; tool_calls: expected ['write_file'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.write_file<|channel|>commentary <|constrain|>json<|message|>{\"path\": \"harmony.md\", \"content\": \"A call ends with <|call|>; a ..., got '; a turn with .\\\\nHeader: {}\"}'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.write_file<|channel|>commentary <|constrain|>json<|message|>{\"path\": \"harmony.md\", \"content\": \"A call ends with <|call|>; a ..., got '; a turn with .\\\\nHeader: {}\"}'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.write_file<|channel|>commentary <|constrain|>json<|message|>{\"path\": \"harmony.md\", \"content\": \"A call ends with <|call|>; a ..., got '; a turn with .\\\\nHeader: {}\"}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.write_file<|channel|>commentary <|constrain|>json<|message|>{\"path\": \"harmony.md\", \"content\": \"A call ends with <|call|>; a ..., got '; a turn with .\\\\nHeader: {}\"}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.write_file<|channel|>commentary <|constrain|>json<|message|>{\"path\": \"harmony.md\", \"content\": \"A call ends with <|call|>; a ..., got '; a turn with .\\\\nHeader: {}\"}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.write_file<|channel|>commentary <|constrain|>json<|message|>{\"path\": \"harmony.md\", \"content\": \"A call ends with <|call|>; a ..., got '; a turn with .\\\\nHeader: {}\"}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.write_file<|channel|>commentary <|constrain|>json<|message|>{\"path\": \"harmony.md\", \"content\": \"A call ends with <|call|>; a ..., got '; a turn with .\\\\nHeader: {}\"}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.write_file<|channel|>commentary <|constrain|>json<|message|>{\"path\": \"harmony.md\", \"content\": \"A call ends with <|call|>; a ..., got '; a turn with .\\\\nHeader: {}\"}'" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '<|start|>'; content contains '<|message|>'; content contains '<|channel|>'; content contains '<|constrain|>'; content contains '<|call|>'" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 5, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] }, "observed": { "nonstream": { "content": "<|start|>assistant to=functions.write_file<|channel|>commentary <|constrain|>json<|message|>{\"path\": \"harmony.md\", \"content\": \"A call ends with <|call|>; a turn with .\\nHeader: {}\"}", "reasoning_content": "Write the Harmony cheat sheet to a file.", "tool_calls": [], "exception": null }, "streams": { "one": { "content": "; a turn with .\\nHeader: {}\"}", "reasoning_content": "Write the Harmony cheat sheet to a file.", "tool_calls": [], "exception": null }, "special": { "content": "; a turn with .\\nHeader: {}\"}", "reasoning_content": "Write the Harmony cheat sheet to a file.", "tool_calls": [], "exception": null }, "token": { "content": "; a turn with .\\nHeader: {}\"}", "reasoning_content": "Write the Harmony cheat sheet to a file.", "tool_calls": [], "exception": null }, "rand:1:8": { "content": "; a turn with .\\nHeader: {}\"}", "reasoning_content": "Write the Harmony cheat sheet to a file.", "tool_calls": [], "exception": null }, "rand:2:8": { "content": "; a turn with .\\nHeader: {}\"}", "reasoning_content": "Write the Harmony cheat sheet to a file.", "tool_calls": [], "exception": null }, "rand:3:8": { "content": "; a turn with .\\nHeader: {}\"}", "reasoning_content": "Write the Harmony cheat sheet to a file.", "tool_calls": [], "exception": null }, "rand:4:8": { "content": "; a turn with .\\nHeader: {}\"}", "reasoning_content": "Write the Harmony cheat sheet to a file.", "tool_calls": [], "exception": null }, "rand:5:8": { "content": "; a turn with .\\nHeader: {}\"}", "reasoning_content": "Write the Harmony cheat sheet to a file.", "tool_calls": [], "exception": null } } } }, { "fixture_id": "gpt-oss/harmony-marker-in-reasoning", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '<|channel|>analysis<|message|>I must not write <|call|> or {}<|call|>'; reasoning_content: expected 'I must not write <|call|> or <|channel|>final myself; the tool call does it.', got None; tool_calls: expected ['get_time'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected None, got ' or {}'; reasoning_content: expected 'I must not write <|call|> or <|channel|>final myself; the tool call does it.', got None; tool_calls: expected ['get_time'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected None, got ' or {}'; reasoning_content: expected 'I must not write <|call|> or <|channel|>final myself; the tool call does it.', got None; tool_calls: expected ['get_time'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls: expected ['get_time'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: expected ['get_time'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: expected ['get_time'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected None, got ' or {}'; reasoning_content: expected 'I must not write <|call|> or <|channel|>final myself; the tool call does it.', got 'I must not write'; tool_calls: expected ['get_time'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: expected ['get_time'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: expected ['get_time'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '<|channel|>analysis<|message|>I must not write <|call|> or {}<|call|>', got ' or {}'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '<|channel|>analysis<|message|>I must not write <|call|> or {}<|call|>', got ' or {}'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '<|channel|>analysis<|message|>I must not write <|call|> or {}<|call|>', got None; reasoning_content: nonstream None, got 'I must not write <|call|> or <|channel|>final myself; the tool call does it.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '<|channel|>analysis<|message|>I must not write <|call|> or {}<|call|>', got None; reasoning_content: nonstream None, got 'I must not write <|call|> or <|channel|>final myself; the tool call does it.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '<|channel|>analysis<|message|>I must not write <|call|> or {}<|call|>', got None; reasoning_content: nonstream None, got 'I must not write <|call|> or <|channel|>final myself; the tool call does it.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '<|channel|>analysis<|message|>I must not write <|call|> or {}<|call|>', got ' or {}'; reasoning_content: nonstream None, got 'I must not write'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '<|channel|>analysis<|message|>I must not write <|call|> or {}<|call|>', got None; reasoning_content: nonstream None, got 'I must not write <|call|> or <|channel|>final myself; the tool call does it.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '<|channel|>analysis<|message|>I must not write <|call|> or {}<|call|>', got None; reasoning_content: nonstream None, got 'I must not write <|call|> or <|channel|>final myself; the tool call does it.'" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content: one ' or {}', got None; reasoning_content: one None, got 'I must not write <|call|> or <|channel|>final myself; the tool call does it.' | rand:1:8 (fail): content: one ' or {}', got None; reasoning_content: one None, got 'I must not write <|call|> or <|channel|>final myself; the tool call does it.' | rand:2:8 (fail): content: one ' or {}', got None; reasoning_content: one None, got 'I must not write <|call|> or <|channel|>final myself; the tool call does it.' | rand:3:8 (fail): reasoning_content: one None, got 'I must not write' | rand:4:8 (fail): content: one ' or {}', got None; reasoning_content: one None, got 'I must not write <|call|> or <|channel|>final myself; the tool call does it.' | rand:5:8 (fail): content: one ' or {}', got None; reasoning_content: one None, got 'I must not write <|call|> or <|channel|>final myself; the tool call does it.'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '<|message|>'; content contains '<|channel|>'; content contains '<|call|>'" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 5, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] }, "observed": { "nonstream": { "content": "<|channel|>analysis<|message|>I must not write <|call|> or {}<|call|>", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": " or {}", "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": " or {}", "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": "I must not write <|call|> or <|channel|>final myself; the tool call does it.", "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "I must not write <|call|> or <|channel|>final myself; the tool call does it.", "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "I must not write <|call|> or <|channel|>final myself; the tool call does it.", "tool_calls": [], "exception": null }, "rand:3:8": { "content": " or {}", "reasoning_content": "I must not write", "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "I must not write <|call|> or <|channel|>final myself; the tool call does it.", "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "I must not write <|call|> or <|channel|>final myself; the tool call does it.", "tool_calls": [], "exception": null } } } }, { "fixture_id": "gpt-oss/harmony-preamble-then-call", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected 'Let me check the weather for you.', got 'Let me check the weather for you.<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\": \"Paris\", \"unit\": \"c\"}...; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected 'Let me check the weather for you.', got 'Let me check the weather for you.Let me check the weather for you.'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected 'Let me check the weather for you.', got 'Let me check the weather for you.Let me check the weather for you.'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected 'Let me check the weather for you.', got 'Let me check the weather for you.Let me check the weather for you.'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected 'Let me check the weather for you.', got 'Let me check the weather for you.Let me check the weather for you.'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected 'Let me check the weather for you.', got 'Let me check the weather for you.Let me check the weather for you.'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected 'Let me check the weather for you.', got 'Let me check the weather for you.Let me check the weather for you.'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected 'Let me check the weather for you.', got 'Let me check the weather for you.Let me check the weather for you.'; tool_calls: expected ['get_weather'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream 'Let me check the weather for you.<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\": \"Paris\", \"unit\": \"c\"}..., got 'Let me check the weather for you.'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream 'Let me check the weather for you.<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\": \"Paris\", \"unit\": \"c\"}..., got 'Let me check the weather for you.Let me check the weather for you.'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream 'Let me check the weather for you.<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\": \"Paris\", \"unit\": \"c\"}..., got 'Let me check the weather for you.Let me check the weather for you.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream 'Let me check the weather for you.<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\": \"Paris\", \"unit\": \"c\"}..., got 'Let me check the weather for you.Let me check the weather for you.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream 'Let me check the weather for you.<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\": \"Paris\", \"unit\": \"c\"}..., got 'Let me check the weather for you.Let me check the weather for you.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream 'Let me check the weather for you.<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\": \"Paris\", \"unit\": \"c\"}..., got 'Let me check the weather for you.Let me check the weather for you.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream 'Let me check the weather for you.<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\": \"Paris\", \"unit\": \"c\"}..., got 'Let me check the weather for you.Let me check the weather for you.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream 'Let me check the weather for you.<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\": \"Paris\", \"unit\": \"c\"}..., got 'Let me check the weather for you.Let me check the weather for you.'" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: special (fail): content: one 'Let me check the weather for you.', got 'Let me check the weather for you.Let me check the weather for you.' | token (fail): content: one 'Let me check the weather for you.', got 'Let me check the weather for you.Let me check the weather for you.' | rand:1:8 (fail): content: one 'Let me check the weather for you.', got 'Let me check the weather for you.Let me check the weather for you.' | rand:2:8 (fail): content: one 'Let me check the weather for you.', got 'Let me check the weather for you.Let me check the weather for you.' | rand:3:8 (fail): content: one 'Let me check the weather for you.', got 'Let me check the weather for you.Let me check the weather for you.' | rand:4:8 (fail): content: one 'Let me check the weather for you.', got 'Let me check the weather for you.Let me check the weather for you.' | rand:5:8 (fail): content: one 'Let me check the weather for you.', got 'Let me check the weather for you.Let me check the weather for you.'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '<|start|>'; content contains '<|message|>'; content contains '<|channel|>'; content contains '<|constrain|>'; content contains '<|call|>'" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 5, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] }, "observed": { "nonstream": { "content": "Let me check the weather for you.<|start|>assistant to=functions.get_weather<|channel|>commentary <|constrain|>json<|message|>{\"city\": \"Paris\", \"unit\": \"c\"}<|call|>", "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "streams": { "one": { "content": "Let me check the weather for you.", "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "special": { "content": "Let me check the weather for you.Let me check the weather for you.", "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "token": { "content": "Let me check the weather for you.Let me check the weather for you.", "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "rand:1:8": { "content": "Let me check the weather for you.Let me check the weather for you.", "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "rand:2:8": { "content": "Let me check the weather for you.Let me check the weather for you.", "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "rand:3:8": { "content": "Let me check the weather for you.Let me check the weather for you.", "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "rand:4:8": { "content": "Let me check the weather for you.Let me check the weather for you.", "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "rand:5:8": { "content": "Let me check the weather for you.Let me check the weather for you.", "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null } } } }, { "fixture_id": "gpt-oss/harmony-preamble-no-reasoning", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected 'Checking two sources — one moment.', got 'Checking two sources — one moment.<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"vLLM release notes\"}<|c...; tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected 'Checking two sources — one moment.', got 'Checking two sources — one moment.Checking two sources — one moment.'; tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected 'Checking two sources — one moment.', got 'Checking two sources — one moment.Checking two sources — one moment.'; tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected 'Checking two sources — one moment.', got 'Checking two sources — one moment.Checking two sources — one moment.'; tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected 'Checking two sources — one moment.', got 'Checking two sources — one moment.Checking two sources — one moment.'; tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected 'Checking two sources — one moment.', got 'Checking two sources — one moment.Checking two sources — one moment.'; tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected 'Checking two sources — one moment.', got 'Checking two sources — one moment.Checking two sources — one moment.'; tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected 'Checking two sources — one moment.', got 'Checking two sources — one moment.Checking two sources — one moment.'; tool_calls: expected ['search'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream 'Checking two sources — one moment.<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"vLLM release notes\"}<|c..., got 'Checking two sources — one moment.'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream 'Checking two sources — one moment.<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"vLLM release notes\"}<|c..., got 'Checking two sources — one moment.Checking two sources — one moment.'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream 'Checking two sources — one moment.<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"vLLM release notes\"}<|c..., got 'Checking two sources — one moment.Checking two sources — one moment.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream 'Checking two sources — one moment.<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"vLLM release notes\"}<|c..., got 'Checking two sources — one moment.Checking two sources — one moment.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream 'Checking two sources — one moment.<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"vLLM release notes\"}<|c..., got 'Checking two sources — one moment.Checking two sources — one moment.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream 'Checking two sources — one moment.<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"vLLM release notes\"}<|c..., got 'Checking two sources — one moment.Checking two sources — one moment.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream 'Checking two sources — one moment.<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"vLLM release notes\"}<|c..., got 'Checking two sources — one moment.Checking two sources — one moment.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream 'Checking two sources — one moment.<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"vLLM release notes\"}<|c..., got 'Checking two sources — one moment.Checking two sources — one moment.'" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: special (fail): content: one 'Checking two sources — one moment.', got 'Checking two sources — one moment.Checking two sources — one moment.' | token (fail): content: one 'Checking two sources — one moment.', got 'Checking two sources — one moment.Checking two sources — one moment.' | rand:1:8 (fail): content: one 'Checking two sources — one moment.', got 'Checking two sources — one moment.Checking two sources — one moment.' | rand:2:8 (fail): content: one 'Checking two sources — one moment.', got 'Checking two sources — one moment.Checking two sources — one moment.' | rand:3:8 (fail): content: one 'Checking two sources — one moment.', got 'Checking two sources — one moment.Checking two sources — one moment.' | rand:4:8 (fail): content: one 'Checking two sources — one moment.', got 'Checking two sources — one moment.Checking two sources — one moment.' | rand:5:8 (fail): content: one 'Checking two sources — one moment.', got 'Checking two sources — one moment.Checking two sources — one moment.'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '<|start|>'; content contains '<|message|>'; content contains '<|channel|>'; content contains '<|constrain|>'; content contains '<|call|>'" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 5, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] }, "observed": { "nonstream": { "content": "Checking two sources — one moment.<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"vLLM release notes\"}<|call|>", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": "Checking two sources — one moment.", "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": "Checking two sources — one moment.Checking two sources — one moment.", "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": "Checking two sources — one moment.Checking two sources — one moment.", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:1:8": { "content": "Checking two sources — one moment.Checking two sources — one moment.", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:2:8": { "content": "Checking two sources — one moment.Checking two sources — one moment.", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:3:8": { "content": "Checking two sources — one moment.Checking two sources — one moment.", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:4:8": { "content": "Checking two sources — one moment.Checking two sources — one moment.", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:5:8": { "content": "Checking two sources — one moment.Checking two sources — one moment.", "reasoning_content": null, "tool_calls": [], "exception": null } } } }, { "fixture_id": "gpt-oss/harmony-no-call-final", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 5, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|return|>", "id": 200002, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gpt-oss/harmony-final-only", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 5, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|return|>", "id": 200002, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gpt-oss/harmony-final-json-lookalike", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 5, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|return|>", "id": 200002, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gpt-oss/harmony-string-escapes", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '<|start|>assistant to=functions.write_file<|channel|>commentary <|constrain|>json<|message|>{\"path\": \"C:\\\\\\\\Users\\\\\\\\me\\\\\\\\notes.txt\", \"content\": \"line 1\\\\n...; tool_calls: expected ['write_file'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['write_file'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['write_file'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls: expected ['write_file'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: expected ['write_file'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: expected ['write_file'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: expected ['write_file'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: expected ['write_file'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: expected ['write_file'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.write_file<|channel|>commentary <|constrain|>json<|message|>{\"path\": \"C:\\\\\\\\Users\\\\\\\\me\\\\\\\\notes.txt\", \"content\": \"line 1\\\\n..., got None" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.write_file<|channel|>commentary <|constrain|>json<|message|>{\"path\": \"C:\\\\\\\\Users\\\\\\\\me\\\\\\\\notes.txt\", \"content\": \"line 1\\\\n..., got None" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.write_file<|channel|>commentary <|constrain|>json<|message|>{\"path\": \"C:\\\\\\\\Users\\\\\\\\me\\\\\\\\notes.txt\", \"content\": \"line 1\\\\n..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.write_file<|channel|>commentary <|constrain|>json<|message|>{\"path\": \"C:\\\\\\\\Users\\\\\\\\me\\\\\\\\notes.txt\", \"content\": \"line 1\\\\n..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.write_file<|channel|>commentary <|constrain|>json<|message|>{\"path\": \"C:\\\\\\\\Users\\\\\\\\me\\\\\\\\notes.txt\", \"content\": \"line 1\\\\n..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.write_file<|channel|>commentary <|constrain|>json<|message|>{\"path\": \"C:\\\\\\\\Users\\\\\\\\me\\\\\\\\notes.txt\", \"content\": \"line 1\\\\n..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.write_file<|channel|>commentary <|constrain|>json<|message|>{\"path\": \"C:\\\\\\\\Users\\\\\\\\me\\\\\\\\notes.txt\", \"content\": \"line 1\\\\n..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.write_file<|channel|>commentary <|constrain|>json<|message|>{\"path\": \"C:\\\\\\\\Users\\\\\\\\me\\\\\\\\notes.txt\", \"content\": \"line 1\\\\n..., got None" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '<|start|>'; content contains '<|message|>'; content contains '<|channel|>'; content contains '<|constrain|>'; content contains '<|call|>'" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 5, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] }, "observed": { "nonstream": { "content": "<|start|>assistant to=functions.write_file<|channel|>commentary <|constrain|>json<|message|>{\"path\": \"C:\\\\Users\\\\me\\\\notes.txt\", \"content\": \"line 1\\n\\t\\\"quoted\\\" and \\\\backslash\\\\\\r\\nend \\u0007\"}<|call|>", "reasoning_content": "Write a Windows path and a quoted line.", "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "Write a Windows path and a quoted line.", "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": "Write a Windows path and a quoted line.", "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": "Write a Windows path and a quoted line.", "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "Write a Windows path and a quoted line.", "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "Write a Windows path and a quoted line.", "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "Write a Windows path and a quoted line.", "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "Write a Windows path and a quoted line.", "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "Write a Windows path and a quoted line.", "tool_calls": [], "exception": null } } } }, { "fixture_id": "gpt-oss/harmony-numeric-arguments", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '<|start|>assistant to=functions.set_alarm<|channel|>commentary <|constrain|>json<|message|>{\"hour\": 7, \"minute\": 30, \"volume\": 0.75, \"repeat\": true, \"label\"...; tool_calls: expected ['set_alarm'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['set_alarm'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['set_alarm'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls: expected ['set_alarm'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: expected ['set_alarm'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: expected ['set_alarm'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: expected ['set_alarm'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: expected ['set_alarm'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: expected ['set_alarm'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.set_alarm<|channel|>commentary <|constrain|>json<|message|>{\"hour\": 7, \"minute\": 30, \"volume\": 0.75, \"repeat\": true, \"label\"..., got None" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.set_alarm<|channel|>commentary <|constrain|>json<|message|>{\"hour\": 7, \"minute\": 30, \"volume\": 0.75, \"repeat\": true, \"label\"..., got None" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.set_alarm<|channel|>commentary <|constrain|>json<|message|>{\"hour\": 7, \"minute\": 30, \"volume\": 0.75, \"repeat\": true, \"label\"..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.set_alarm<|channel|>commentary <|constrain|>json<|message|>{\"hour\": 7, \"minute\": 30, \"volume\": 0.75, \"repeat\": true, \"label\"..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.set_alarm<|channel|>commentary <|constrain|>json<|message|>{\"hour\": 7, \"minute\": 30, \"volume\": 0.75, \"repeat\": true, \"label\"..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.set_alarm<|channel|>commentary <|constrain|>json<|message|>{\"hour\": 7, \"minute\": 30, \"volume\": 0.75, \"repeat\": true, \"label\"..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.set_alarm<|channel|>commentary <|constrain|>json<|message|>{\"hour\": 7, \"minute\": 30, \"volume\": 0.75, \"repeat\": true, \"label\"..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.set_alarm<|channel|>commentary <|constrain|>json<|message|>{\"hour\": 7, \"minute\": 30, \"volume\": 0.75, \"repeat\": true, \"label\"..., got None" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '<|start|>'; content contains '<|message|>'; content contains '<|channel|>'; content contains '<|constrain|>'; content contains '<|call|>'" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 5, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] }, "observed": { "nonstream": { "content": "<|start|>assistant to=functions.set_alarm<|channel|>commentary <|constrain|>json<|message|>{\"hour\": 7, \"minute\": 30, \"volume\": 0.75, \"repeat\": true, \"label\": null}<|call|>", "reasoning_content": "Set the alarm.", "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "Set the alarm.", "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": "Set the alarm.", "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": "Set the alarm.", "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "Set the alarm.", "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "Set the alarm.", "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "Set the alarm.", "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "Set the alarm.", "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "Set the alarm.", "tool_calls": [], "exception": null } } } }, { "fixture_id": "gpt-oss/harmony-long-arguments", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '<|start|>assistant to=functions.write_file<|channel|>commentary <|constrain|>json<|message|>{\"path\": \"steps.py\", \"content\": \"\\\\\"\\\\\"\\\\\"Generated module used ...; tool_calls: expected ['write_file'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['write_file'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['write_file'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls: expected ['write_file'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: expected ['write_file'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: expected ['write_file'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: expected ['write_file'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: expected ['write_file'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: expected ['write_file'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.write_file<|channel|>commentary <|constrain|>json<|message|>{\"path\": \"steps.py\", \"content\": \"\\\\\"\\\\\"\\\\\"Generated module used ..., got None" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.write_file<|channel|>commentary <|constrain|>json<|message|>{\"path\": \"steps.py\", \"content\": \"\\\\\"\\\\\"\\\\\"Generated module used ..., got None" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.write_file<|channel|>commentary <|constrain|>json<|message|>{\"path\": \"steps.py\", \"content\": \"\\\\\"\\\\\"\\\\\"Generated module used ..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.write_file<|channel|>commentary <|constrain|>json<|message|>{\"path\": \"steps.py\", \"content\": \"\\\\\"\\\\\"\\\\\"Generated module used ..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.write_file<|channel|>commentary <|constrain|>json<|message|>{\"path\": \"steps.py\", \"content\": \"\\\\\"\\\\\"\\\\\"Generated module used ..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.write_file<|channel|>commentary <|constrain|>json<|message|>{\"path\": \"steps.py\", \"content\": \"\\\\\"\\\\\"\\\\\"Generated module used ..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.write_file<|channel|>commentary <|constrain|>json<|message|>{\"path\": \"steps.py\", \"content\": \"\\\\\"\\\\\"\\\\\"Generated module used ..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.write_file<|channel|>commentary <|constrain|>json<|message|>{\"path\": \"steps.py\", \"content\": \"\\\\\"\\\\\"\\\\\"Generated module used ..., got None" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '<|start|>'; content contains '<|message|>'; content contains '<|channel|>'; content contains '<|constrain|>'; content contains '<|call|>'" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 5, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] }, "observed": { "nonstream": { "content": "<|start|>assistant to=functions.write_file<|channel|>commentary <|constrain|>json<|message|>{\"path\": \"steps.py\", \"content\": \"\\\"\\\"\\\"Generated module used as a long tool argument.\\\"\\\"\\\"\\n\\nfrom __future__ import annotations\\n\\ndef step_00(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 0.\\\"\\\"\\\"\\n return x + 0\\n\\ndef step_01(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 1.\\\"\\\"\\\"\\n return x + 1\\n\\ndef step_02(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 2.\\\"\\\"\\\"\\n return x + 2\\n\\ndef step_03(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 3.\\\"\\\"\\\"\\n return x + 3\\n\\ndef step_04(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 4.\\\"\\\"\\\"\\n return x + 4\\n\\ndef step_05(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 5.\\\"\\\"\\\"\\n return x + 5\\n\\ndef step_06(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 6.\\\"\\\"\\\"\\n return x + 6\\n\\ndef step_07(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 7.\\\"\\\"\\\"\\n return x + 7\\n\\ndef step_08(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 8.\\\"\\\"\\\"\\n return x + 8\\n\\ndef step_09(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 9.\\\"\\\"\\\"\\n return x + 9\\n\\ndef step_10(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 10.\\\"\\\"\\\"\\n return x + 10\\n\\ndef step_11(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 11.\\\"\\\"\\\"\\n return x + 11\\n\\ndef step_12(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 12.\\\"\\\"\\\"\\n return x + 12\\n\\ndef step_13(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 13.\\\"\\\"\\\"\\n return x + 13\\n\\ndef step_14(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 14.\\\"\\\"\\\"\\n return x + 14\\n\\ndef step_15(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 15.\\\"\\\"\\\"\\n return x + 15\\n\\ndef step_16(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 16.\\\"\\\"\\\"\\n return x + 16\\n\\ndef step_17(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 17.\\\"\\\"\\\"\\n return x + 17\\n\\ndef step_18(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 18.\\\"\\\"\\\"\\n return x + 18\\n\\ndef step_19(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 19.\\\"\\\"\\\"\\n return x + 19\\n\\ndef step_20(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 20.\\\"\\\"\\\"\\n return x + 20\\n\\ndef step_21(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 21.\\\"\\\"\\\"\\n return x + 21\\n\\ndef step_22(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 22.\\\"\\\"\\\"\\n return x + 22\\n\\ndef step_23(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 23.\\\"\\\"\\\"\\n return x + 23\\n\\ndef step_24(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 24.\\\"\\\"\\\"\\n return x + 24\\n\\ndef step_25(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 25.\\\"\\\"\\\"\\n return x + 25\\n\\ndef step_26(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 26.\\\"\\\"\\\"\\n return x + 26\\n\\ndef step_27(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 27.\\\"\\\"\\\"\\n return x + 27\\n\\ndef step_28(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 28.\\\"\\\"\\\"\\n return x + 28\\n\\ndef step_29(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 29.\\\"\\\"\\\"\\n return x + 29\\n\\ndef step_30(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 30.\\\"\\\"\\\"\\n return x + 30\\n\\ndef step_31(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 31.\\\"\\\"\\\"\\n return x + 31\\n\\ndef step_32(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 32.\\\"\\\"\\\"\\n return x + 32\\n\\ndef step_33(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 33.\\\"\\\"\\\"\\n return x + 33\\n\\ndef step_34(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 34.\\\"\\\"\\\"\\n return x + 34\\n\\ndef step_35(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 35.\\\"\\\"\\\"\\n return x + 35\\n\\ndef step_36(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 36.\\\"\\\"\\\"\\n return x + 36\\n\\ndef step_37(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 37.\\\"\\\"\\\"\\n return x + 37\\n\\ndef step_38(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 38.\\\"\\\"\\\"\\n return x + 38\\n\\ndef step_39(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 39.\\\"\\\"\\\"\\n return x + 39\\n\\ndef step_40(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 40.\\\"\\\"\\\"\\n return x + 40\\n\\ndef step_41(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 41.\\\"\\\"\\\"\\n return x + 41\\n\\ndef step_42(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 42.\\\"\\\"\\\"\\n return x + 42\\n\\ndef step_43(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 43.\\\"\\\"\\\"\\n return x + 43\\n\\ndef step_44(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 44.\\\"\\\"\\\"\\n return x + 44\\n\\ndef step_45(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 45.\\\"\\\"\\\"\\n return x + 45\\n\\ndef step_46(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 46.\\\"\\\"\\\"\\n return x + 46\\n\\ndef step_47(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 47.\\\"\\\"\\\"\\n return x + 47\\n\\ndef step_48(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 48.\\\"\\\"\\\"\\n return x + 48\\n\\ndef step_49(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 49.\\\"\\\"\\\"\\n return x + 49\\n\\ndef step_50(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 50.\\\"\\\"\\\"\\n return x + 50\\n\\ndef step_51(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 51.\\\"\\\"\\\"\\n return x + 51\\n\\ndef step_52(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 52.\\\"\\\"\\\"\\n return x + 52\\n\\ndef step_53(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 53.\\\"\\\"\\\"\\n return x + 53\\n\\ndef step_54(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 54.\\\"\\\"\\\"\\n return x + 54\\n\\ndef step_55(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 55.\\\"\\\"\\\"\\n return x + 55\\n\\ndef step_56(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 56.\\\"\\\"\\\"\\n return x + 56\\n\\ndef step_57(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 57.\\\"\\\"\\\"\\n return x + 57\\n\\ndef step_58(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 58.\\\"\\\"\\\"\\n return x + 58\\n\\ndef step_59(x: int) -> int:\\n \\\"\\\"\\\"Return x plus 59.\\\"\\\"\\\"\\n return x + 59\\n\"}<|call|>", "reasoning_content": "Write the module.", "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "Write the module.", "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": "Write the module.", "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": "Write the module.", "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "Write the module.", "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "Write the module.", "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "Write the module.", "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "Write the module.", "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "Write the module.", "tool_calls": [], "exception": null } } } }, { "fixture_id": "gpt-oss/harmony-multiline-reasoning", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"weather Paris\"}<|call|>'; tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"weather Paris\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"weather Paris\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"weather Paris\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"weather Paris\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"weather Paris\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"weather Paris\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"weather Paris\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"weather Paris\"}<|call|>', got None" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '<|start|>'; content contains '<|message|>'; content contains '<|channel|>'; content contains '<|constrain|>'; content contains '<|call|>'" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 5, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] }, "observed": { "nonstream": { "content": "<|start|>assistant to=functions.search<|channel|>commentary <|constrain|>json<|message|>{\"query\": \"weather Paris\"}<|call|>", "reasoning_content": "Step 1: parse the request.\nStep 2: it needs live data.\n\nStep 3: call search.", "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "Step 1: parse the request.\nStep 2: it needs live data.\n\nStep 3: call search.", "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": "Step 1: parse the request.\nStep 2: it needs live data.\n\nStep 3: call search.", "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": "Step 1: parse the request.\nStep 2: it needs live data.\n\nStep 3: call search.", "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "Step 1: parse the request.\nStep 2: it needs live data.\n\nStep 3: call search.", "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "Step 1: parse the request.\nStep 2: it needs live data.\n\nStep 3: call search.", "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "Step 1: parse the request.\nStep 2: it needs live data.\n\nStep 3: call search.", "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "Step 1: parse the request.\nStep 2: it needs live data.\n\nStep 3: call search.", "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "Step 1: parse the request.\nStep 2: it needs live data.\n\nStep 3: call search.", "tool_calls": [], "exception": null } } } }, { "fixture_id": "gpt-oss/harmony-content-type-json-text", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '<|start|>assistant to=functions.get_weather<|channel|>commentary json<|message|>{\"city\": \"Paris\"}<|call|>'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary json<|message|>{\"city\": \"Paris\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary json<|message|>{\"city\": \"Paris\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary json<|message|>{\"city\": \"Paris\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary json<|message|>{\"city\": \"Paris\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary json<|message|>{\"city\": \"Paris\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary json<|message|>{\"city\": \"Paris\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary json<|message|>{\"city\": \"Paris\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary json<|message|>{\"city\": \"Paris\"}<|call|>', got None" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '<|start|>'; content contains '<|message|>'; content contains '<|channel|>'; content contains '<|call|>'" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 5, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] }, "observed": { "nonstream": { "content": "<|start|>assistant to=functions.get_weather<|channel|>commentary json<|message|>{\"city\": \"Paris\"}<|call|>", "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null } } } }, { "fixture_id": "gpt-oss/harmony-no-content-type", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '<|start|>assistant to=functions.get_weather<|channel|>commentary<|message|>{\"city\": \"Paris\"}<|call|>'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary<|message|>{\"city\": \"Paris\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary<|message|>{\"city\": \"Paris\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary<|message|>{\"city\": \"Paris\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary<|message|>{\"city\": \"Paris\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary<|message|>{\"city\": \"Paris\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary<|message|>{\"city\": \"Paris\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary<|message|>{\"city\": \"Paris\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary<|message|>{\"city\": \"Paris\"}<|call|>', got None" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '<|start|>'; content contains '<|message|>'; content contains '<|channel|>'; content contains '<|call|>'" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 5, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] }, "observed": { "nonstream": { "content": "<|start|>assistant to=functions.get_weather<|channel|>commentary<|message|>{\"city\": \"Paris\"}<|call|>", "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null } } } }, { "fixture_id": "gpt-oss/harmony-truncated-in-arguments", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 5, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gpt-oss/harmony-truncated-in-reasoning", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 5, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gpt-oss/harmony-truncated-in-header", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 5, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "gpt-oss/hf-template-single-call", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '<|start|>assistant to=functions.get_weather<|channel|>commentary json<|message|>{\"city\": \"Paris\", \"unit\": \"c\"}<|call|>'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary json<|message|>{\"city\": \"Paris\", \"unit\": \"c\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary json<|message|>{\"city\": \"Paris\", \"unit\": \"c\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary json<|message|>{\"city\": \"Paris\", \"unit\": \"c\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary json<|message|>{\"city\": \"Paris\", \"unit\": \"c\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary json<|message|>{\"city\": \"Paris\", \"unit\": \"c\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary json<|message|>{\"city\": \"Paris\", \"unit\": \"c\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary json<|message|>{\"city\": \"Paris\", \"unit\": \"c\"}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_weather<|channel|>commentary json<|message|>{\"city\": \"Paris\", \"unit\": \"c\"}<|call|>', got None" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '<|start|>'; content contains '<|message|>'; content contains '<|channel|>'; content contains '<|call|>'" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 5, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] }, "observed": { "nonstream": { "content": "<|start|>assistant to=functions.get_weather<|channel|>commentary json<|message|>{\"city\": \"Paris\", \"unit\": \"c\"}<|call|>", "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "The user asks for the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null } } } }, { "fixture_id": "gpt-oss/hf-template-unicode-nested", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '<|start|>assistant to=functions.search<|channel|>commentary json<|message|>{\"query\": \"café \\\\\"best\\\\\" Zürich ☕\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}<...; tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary json<|message|>{\"query\": \"café \\\\\"best\\\\\" Zürich ☕\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}<..., got None" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary json<|message|>{\"query\": \"café \\\\\"best\\\\\" Zürich ☕\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}<..., got None" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary json<|message|>{\"query\": \"café \\\\\"best\\\\\" Zürich ☕\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}<..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary json<|message|>{\"query\": \"café \\\\\"best\\\\\" Zürich ☕\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}<..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary json<|message|>{\"query\": \"café \\\\\"best\\\\\" Zürich ☕\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}<..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary json<|message|>{\"query\": \"café \\\\\"best\\\\\" Zürich ☕\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}<..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary json<|message|>{\"query\": \"café \\\\\"best\\\\\" Zürich ☕\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}<..., got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.search<|channel|>commentary json<|message|>{\"query\": \"café \\\\\"best\\\\\" Zürich ☕\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}<..., got None" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '<|start|>'; content contains '<|message|>'; content contains '<|channel|>'; content contains '<|call|>'" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 5, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] }, "observed": { "nonstream": { "content": "<|start|>assistant to=functions.search<|channel|>commentary json<|message|>{\"query\": \"café \\\"best\\\" Zürich ☕\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}<|call|>", "reasoning_content": "Suche nach dem besten Café in Zürich.", "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "Suche nach dem besten Café in Zürich.", "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": "Suche nach dem besten Café in Zürich.", "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": "Suche nach dem besten Café in Zürich.", "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "Suche nach dem besten Café in Zürich.", "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "Suche nach dem besten Café in Zürich.", "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "Suche nach dem besten Café in Zürich.", "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "Suche nach dem besten Café in Zürich.", "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "Suche nach dem besten Café in Zürich.", "tool_calls": [], "exception": null } } } }, { "fixture_id": "gpt-oss/hf-template-empty-arguments", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '<|start|>assistant to=functions.get_time<|channel|>commentary json<|message|>{}<|call|>'; tool_calls: expected ['get_time'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_time'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['get_time'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls: expected ['get_time'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: expected ['get_time'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: expected ['get_time'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: expected ['get_time'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: expected ['get_time'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: expected ['get_time'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_time<|channel|>commentary json<|message|>{}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_time<|channel|>commentary json<|message|>{}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_time<|channel|>commentary json<|message|>{}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_time<|channel|>commentary json<|message|>{}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_time<|channel|>commentary json<|message|>{}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_time<|channel|>commentary json<|message|>{}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_time<|channel|>commentary json<|message|>{}<|call|>', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '<|start|>assistant to=functions.get_time<|channel|>commentary json<|message|>{}<|call|>', got None" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '<|start|>'; content contains '<|message|>'; content contains '<|channel|>'; content contains '<|call|>'" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 5, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] }, "observed": { "nonstream": { "content": "<|start|>assistant to=functions.get_time<|channel|>commentary json<|message|>{}<|call|>", "reasoning_content": "Get the time.", "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "Get the time.", "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": "Get the time.", "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": "Get the time.", "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "Get the time.", "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "Get the time.", "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "Get the time.", "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "Get the time.", "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "Get the time.", "tool_calls": [], "exception": null } } } }, { "fixture_id": "gpt-oss/hf-template-no-reasoning", "family": "gpt-oss", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got ' to=functions.set_alarm<|channel|>commentary json<|message|>{\"hour\": 6, \"minute\": 45, \"repeat\": false}<|call|>'; tool_calls: expected ['set_alarm'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected None, got ' to=functions.set_alarm'; tool_calls: expected ['set_alarm'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected None, got ' to=functions.set_alarm to=functions.set_alarm'; tool_calls: expected ['set_alarm'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected None, got ' to=functions.set_alarm to=functions.set_alarm'; tool_calls: expected ['set_alarm'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected None, got ' to=functions.set_alarm to=functions.set_alarm'; tool_calls: expected ['set_alarm'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected None, got ' to=functions.set_alarm to=functions.set_alarm'; tool_calls: expected ['set_alarm'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected None, got ' to=functions.set_alarm to=functions.set_alarm'; tool_calls: expected ['set_alarm'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected None, got ' to=functions.set_alarm to=functions.set_alarm'; tool_calls: expected ['set_alarm'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected None, got ' to=functions.set_alarm to=functions.set_alarm'; tool_calls: expected ['set_alarm'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream ' to=functions.set_alarm<|channel|>commentary json<|message|>{\"hour\": 6, \"minute\": 45, \"repeat\": false}<|call|>', got ' to=functions.set_alarm'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream ' to=functions.set_alarm<|channel|>commentary json<|message|>{\"hour\": 6, \"minute\": 45, \"repeat\": false}<|call|>', got ' to=functions.set_alarm to=functions.set_alarm'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream ' to=functions.set_alarm<|channel|>commentary json<|message|>{\"hour\": 6, \"minute\": 45, \"repeat\": false}<|call|>', got ' to=functions.set_alarm to=functions.set_alarm'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream ' to=functions.set_alarm<|channel|>commentary json<|message|>{\"hour\": 6, \"minute\": 45, \"repeat\": false}<|call|>', got ' to=functions.set_alarm to=functions.set_alarm'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream ' to=functions.set_alarm<|channel|>commentary json<|message|>{\"hour\": 6, \"minute\": 45, \"repeat\": false}<|call|>', got ' to=functions.set_alarm to=functions.set_alarm'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream ' to=functions.set_alarm<|channel|>commentary json<|message|>{\"hour\": 6, \"minute\": 45, \"repeat\": false}<|call|>', got ' to=functions.set_alarm to=functions.set_alarm'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream ' to=functions.set_alarm<|channel|>commentary json<|message|>{\"hour\": 6, \"minute\": 45, \"repeat\": false}<|call|>', got ' to=functions.set_alarm to=functions.set_alarm'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream ' to=functions.set_alarm<|channel|>commentary json<|message|>{\"hour\": 6, \"minute\": 45, \"repeat\": false}<|call|>', got ' to=functions.set_alarm to=functions.set_alarm'" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: special (fail): content: one ' to=functions.set_alarm', got ' to=functions.set_alarm to=functions.set_alarm' | token (fail): content: one ' to=functions.set_alarm', got ' to=functions.set_alarm to=functions.set_alarm' | rand:1:8 (fail): content: one ' to=functions.set_alarm', got ' to=functions.set_alarm to=functions.set_alarm' | rand:2:8 (fail): content: one ' to=functions.set_alarm', got ' to=functions.set_alarm to=functions.set_alarm' | rand:3:8 (fail): content: one ' to=functions.set_alarm', got ' to=functions.set_alarm to=functions.set_alarm' | rand:4:8 (fail): content: one ' to=functions.set_alarm', got ' to=functions.set_alarm to=functions.set_alarm' | rand:5:8 (fail): content: one ' to=functions.set_alarm', got ' to=functions.set_alarm to=functions.set_alarm'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '<|message|>'; content contains '<|channel|>'; content contains '<|call|>'" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 5, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|call|>", "id": 200012, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": true }, "notes": [] }, "observed": { "nonstream": { "content": " to=functions.set_alarm<|channel|>commentary json<|message|>{\"hour\": 6, \"minute\": 45, \"repeat\": false}<|call|>", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": " to=functions.set_alarm", "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": " to=functions.set_alarm to=functions.set_alarm", "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": " to=functions.set_alarm to=functions.set_alarm", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:1:8": { "content": " to=functions.set_alarm to=functions.set_alarm", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:2:8": { "content": " to=functions.set_alarm to=functions.set_alarm", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:3:8": { "content": " to=functions.set_alarm to=functions.set_alarm", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:4:8": { "content": " to=functions.set_alarm to=functions.set_alarm", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:5:8": { "content": " to=functions.set_alarm to=functions.set_alarm", "reasoning_content": null, "tool_calls": [], "exception": null } } } }, { "fixture_id": "gpt-oss/hf-template-final-answer", "family": "gpt-oss", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss", "tool_call_detector": "GptOssDetector", "reasoning_detector": "GptOssDetector", "auto_detected": { "tool_call_parser": "gpt-oss", "reasoning_parser": "gpt-oss" }, "model": "openai/gpt-oss-20b", "tokenizer": { "repo": "openai/gpt-oss-20b", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "GptOssForCausalLM" ], "model_type": "gpt_oss" }, "hf_config_error": null, "chat_template_sha256": "a4c9919cbbd4acdd51ccffe22da049264b1b73e59055fa58811a99efbd7c8146", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 5, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200006, 173781 ] }, "stop": { "token": "<|return|>", "id": 200002, "rule": "harmony: <|call|> after a message with a recipient, else <|return|>", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k2i-vllm-single-call-content-before", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k2", "reasoning_parser": null, "tool_call_detector": "KimiK2Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2" }, "model": "moonshotai/Kimi-K2-Instruct-0905", "tokenizer": { "repo": "moonshotai/Kimi-K2-Instruct-0905", "revision": "ac6c49f04883bd0a0598b790693a72061c676629", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "kimi_k2" }, "hf_config_error": null, "chat_template_sha256": "00938c355ca7195fd4f9c923e93af063846d4d5005e06999274f8db57b65bb0b", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 163588, 69702, 163601 ] }, "stop": { "token": "<|im_end|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k2i-vllm-three-tool-calls", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k2", "reasoning_parser": null, "tool_call_detector": "KimiK2Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2" }, "model": "moonshotai/Kimi-K2-Instruct-0905", "tokenizer": { "repo": "moonshotai/Kimi-K2-Instruct-0905", "revision": "ac6c49f04883bd0a0598b790693a72061c676629", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "kimi_k2" }, "hf_config_error": null, "chat_template_sha256": "00938c355ca7195fd4f9c923e93af063846d4d5005e06999274f8db57b65bb0b", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 3, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 163588, 69702, 163601 ] }, "stop": { "token": "<|im_end|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k2i-vllm-multiline-json", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k2", "reasoning_parser": null, "tool_call_detector": "KimiK2Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2" }, "model": "moonshotai/Kimi-K2-Instruct-0905", "tokenizer": { "repo": "moonshotai/Kimi-K2-Instruct-0905", "revision": "ac6c49f04883bd0a0598b790693a72061c676629", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "kimi_k2" }, "hf_config_error": null, "chat_template_sha256": "00938c355ca7195fd4f9c923e93af063846d4d5005e06999274f8db57b65bb0b", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 163588, 69702, 163601 ] }, "stop": { "token": "<|im_end|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k2i-vllm-id-without-functions-prefix", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k2", "reasoning_parser": null, "tool_call_detector": "KimiK2Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2" }, "model": "moonshotai/Kimi-K2-Instruct-0905", "tokenizer": { "repo": "moonshotai/Kimi-K2-Instruct-0905", "revision": "ac6c49f04883bd0a0598b790693a72061c676629", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "kimi_k2" }, "hf_config_error": null, "chat_template_sha256": "00938c355ca7195fd4f9c923e93af063846d4d5005e06999274f8db57b65bb0b", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 163588, 69702, 163601 ] }, "stop": { "token": "<|im_end|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k2i-vllm-noise-between-markers", "family": "kimi", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected 'Reasoning. ', got 'Reasoning. spurious noise '" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected 'Reasoning. ', got 'Reasoning. spurious noise '" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected 'Reasoning. ', got 'Reasoning. spurious noise '" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected 'Reasoning. ', got 'Reasoning. spurious noise '" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected 'Reasoning. ', got 'Reasoning. spurious noise '" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected 'Reasoning. ', got 'Reasoning. spurious noise '" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected 'Reasoning. ', got 'Reasoning. spurious noise '" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected 'Reasoning. ', got 'Reasoning. spurious noise '" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream 'Reasoning. ', got 'Reasoning. spurious noise '" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream 'Reasoning. ', got 'Reasoning. spurious noise '" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream 'Reasoning. ', got 'Reasoning. spurious noise '" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream 'Reasoning. ', got 'Reasoning. spurious noise '" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream 'Reasoning. ', got 'Reasoning. spurious noise '" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream 'Reasoning. ', got 'Reasoning. spurious noise '" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream 'Reasoning. ', got 'Reasoning. spurious noise '" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream 'Reasoning. ', got 'Reasoning. spurious noise '" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k2", "reasoning_parser": null, "tool_call_detector": "KimiK2Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2" }, "model": "moonshotai/Kimi-K2-Instruct-0905", "tokenizer": { "repo": "moonshotai/Kimi-K2-Instruct-0905", "revision": "ac6c49f04883bd0a0598b790693a72061c676629", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "kimi_k2" }, "hf_config_error": null, "chat_template_sha256": "00938c355ca7195fd4f9c923e93af063846d4d5005e06999274f8db57b65bb0b", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 163588, 69702, 163601 ] }, "stop": { "token": "<|im_end|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "Reasoning. ", "reasoning_content": null, "tool_calls": [ { "name": "test", "arguments_raw": "{\"k\": \"v\"}" } ], "exception": null }, "streams": { "one": { "content": "Reasoning. spurious noise ", "reasoning_content": null, "tool_calls": [ { "name": "test", "arguments_raw": "{\"k\": \"v\"} " } ], "exception": null }, "special": { "content": "Reasoning. spurious noise ", "reasoning_content": null, "tool_calls": [ { "name": "test", "arguments_raw": "{\"k\": \"v\"} " } ], "exception": null }, "token": { "content": "Reasoning. spurious noise ", "reasoning_content": null, "tool_calls": [ { "name": "test", "arguments_raw": "{\"k\": \"v\"} " } ], "exception": null }, "rand:1:8": { "content": "Reasoning. spurious noise ", "reasoning_content": null, "tool_calls": [ { "name": "test", "arguments_raw": "{\"k\": \"v\"} " } ], "exception": null }, "rand:2:8": { "content": "Reasoning. spurious noise ", "reasoning_content": null, "tool_calls": [ { "name": "test", "arguments_raw": "{\"k\": \"v\"} " } ], "exception": null }, "rand:3:8": { "content": "Reasoning. spurious noise ", "reasoning_content": null, "tool_calls": [ { "name": "test", "arguments_raw": "{\"k\": \"v\"} " } ], "exception": null }, "rand:4:8": { "content": "Reasoning. spurious noise ", "reasoning_content": null, "tool_calls": [ { "name": "test", "arguments_raw": "{\"k\": \"v\"} " } ], "exception": null }, "rand:5:8": { "content": "Reasoning. spurious noise ", "reasoning_content": null, "tool_calls": [ { "name": "test", "arguments_raw": "{\"k\": \"v\"} " } ], "exception": null } } } }, { "fixture_id": "kimi/k2i-vllm-empty-tool-section", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k2", "reasoning_parser": null, "tool_call_detector": "KimiK2Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2" }, "model": "moonshotai/Kimi-K2-Instruct-0905", "tokenizer": { "repo": "moonshotai/Kimi-K2-Instruct-0905", "revision": "ac6c49f04883bd0a0598b790693a72061c676629", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "kimi_k2" }, "hf_config_error": null, "chat_template_sha256": "00938c355ca7195fd4f9c923e93af063846d4d5005e06999274f8db57b65bb0b", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 163588, 69702, 163601 ] }, "stop": { "token": "<|im_end|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k2i-vllm-content-after-tool-section", "family": "kimi", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected 'Before. After tools.', got 'Before. '" }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream 'Before. ', got 'Before. After tools.'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream 'Before. ', got 'Before. After tools.'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream 'Before. ', got 'Before. After tools.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream 'Before. ', got 'Before. After tools.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream 'Before. ', got 'Before. After tools.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream 'Before. ', got 'Before. After tools.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream 'Before. ', got 'Before. After tools.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream 'Before. ', got 'Before. After tools.'" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k2", "reasoning_parser": null, "tool_call_detector": "KimiK2Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2" }, "model": "moonshotai/Kimi-K2-Instruct-0905", "tokenizer": { "repo": "moonshotai/Kimi-K2-Instruct-0905", "revision": "ac6c49f04883bd0a0598b790693a72061c676629", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "kimi_k2" }, "hf_config_error": null, "chat_template_sha256": "00938c355ca7195fd4f9c923e93af063846d4d5005e06999274f8db57b65bb0b", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 163588, 69702, 163601 ] }, "stop": { "token": "<|im_end|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "Before. ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null }, "streams": { "one": { "content": "Before. After tools.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"} " } ], "exception": null }, "special": { "content": "Before. After tools.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"} " } ], "exception": null }, "token": { "content": "Before. After tools.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"} " } ], "exception": null }, "rand:1:8": { "content": "Before. After tools.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"} " } ], "exception": null }, "rand:2:8": { "content": "Before. After tools.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"} " } ], "exception": null }, "rand:3:8": { "content": "Before. After tools.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"} " } ], "exception": null }, "rand:4:8": { "content": "Before. After tools.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"} " } ], "exception": null }, "rand:5:8": { "content": "Before. After tools.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"} " } ], "exception": null } } } }, { "fixture_id": "kimi/k3-vllm-no-call-thinking-disabled", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k3", "reasoning_parser": "kimi_k3", "tool_call_detector": "KimiK3Detector", "reasoning_detector": "KimiK3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "moonshotai/Kimi-K3", "tokenizer": { "repo": "moonshotai/Kimi-K3", "revision": "f831ab66814297da540d832a5235f8e904f29d06", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK3ForConditionalGeneration" ], "model_type": "kimi_k3" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "kimi_k3", "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1, 163589, 163587, 12092, 163589 ] }, "stop": { "token": "<|end_of_msg|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k3-vllm-call-thinking-disabled", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k3", "reasoning_parser": "kimi_k3", "tool_call_detector": "KimiK3Detector", "reasoning_detector": "KimiK3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "moonshotai/Kimi-K3", "tokenizer": { "repo": "moonshotai/Kimi-K3", "revision": "f831ab66814297da540d832a5235f8e904f29d06", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK3ForConditionalGeneration" ], "model_type": "kimi_k3" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "kimi_k3", "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1, 163589, 163587, 12092, 163589 ] }, "stop": { "token": "<|end_of_msg|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k3-vllm-truncated-tools", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k3", "reasoning_parser": "kimi_k3", "tool_call_detector": "KimiK3Detector", "reasoning_detector": "KimiK3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "moonshotai/Kimi-K3", "tokenizer": { "repo": "moonshotai/Kimi-K3", "revision": "f831ab66814297da540d832a5235f8e904f29d06", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK3ForConditionalGeneration" ], "model_type": "kimi_k3" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "kimi_k3", "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1, 163589, 163587, 12092, 163589 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k3-vllm-reasoning-prefix-consumed", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k3", "reasoning_parser": "kimi_k3", "tool_call_detector": "KimiK3Detector", "reasoning_detector": "KimiK3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "moonshotai/Kimi-K3", "tokenizer": { "repo": "moonshotai/Kimi-K3", "revision": "f831ab66814297da540d832a5235f8e904f29d06", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK3ForConditionalGeneration" ], "model_type": "kimi_k3" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "kimi_k3", "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1, 163589, 163587, 39964, 163589 ] }, "stop": { "token": "<|end_of_msg|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k3-bug-truncated-reasoning-recorded", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k3", "reasoning_parser": "kimi_k3", "tool_call_detector": "KimiK3Detector", "reasoning_detector": "KimiK3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "moonshotai/Kimi-K3", "tokenizer": { "repo": "moonshotai/Kimi-K3", "revision": "f831ab66814297da540d832a5235f8e904f29d06", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK3ForConditionalGeneration" ], "model_type": "kimi_k3" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "kimi_k3", "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 0, "tool_choice": "none", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1, 163589, 163587, 39964, 163589 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k3-bug-response-only-completion", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k3", "reasoning_parser": "kimi_k3", "tool_call_detector": "KimiK3Detector", "reasoning_detector": "KimiK3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "moonshotai/Kimi-K3", "tokenizer": { "repo": "moonshotai/Kimi-K3", "revision": "f831ab66814297da540d832a5235f8e904f29d06", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK3ForConditionalGeneration" ], "model_type": "kimi_k3" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "kimi_k3", "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1, 163589, 163587, 39964, 163589 ] }, "stop": { "token": "<|end_of_msg|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k26-single-call", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2", "tool_call_detector": "KimiK2Detector", "reasoning_detector": "KimiK2Detector", "auto_detected": { "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2" }, "model": "moonshotai/Kimi-K2.6", "tokenizer": { "repo": "moonshotai/Kimi-K2.6", "revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK25ForConditionalGeneration" ], "model_type": "kimi_k25" }, "hf_config_error": null, "chat_template_sha256": "8bf859698fd4781c0e1e1c63ce74422aab27e53ccc5f47116317c64cda06132f", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 163588, 69702, 163601, 163606 ] }, "stop": { "token": "<|im_end|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k26-parallel-same-tool", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2", "tool_call_detector": "KimiK2Detector", "reasoning_detector": "KimiK2Detector", "auto_detected": { "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2" }, "model": "moonshotai/Kimi-K2.6", "tokenizer": { "repo": "moonshotai/Kimi-K2.6", "revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK25ForConditionalGeneration" ], "model_type": "kimi_k25" }, "hf_config_error": null, "chat_template_sha256": "8bf859698fd4781c0e1e1c63ce74422aab27e53ccc5f47116317c64cda06132f", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 163588, 69702, 163601, 163606 ] }, "stop": { "token": "<|im_end|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k26-parallel-three-tools", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2", "tool_call_detector": "KimiK2Detector", "reasoning_detector": "KimiK2Detector", "auto_detected": { "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2" }, "model": "moonshotai/Kimi-K2.6", "tokenizer": { "repo": "moonshotai/Kimi-K2.6", "revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK25ForConditionalGeneration" ], "model_type": "kimi_k25" }, "hf_config_error": null, "chat_template_sha256": "8bf859698fd4781c0e1e1c63ce74422aab27e53ccc5f47116317c64cda06132f", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 163588, 69702, 163601, 163606 ] }, "stop": { "token": "<|im_end|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k26-content-before-call", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2", "tool_call_detector": "KimiK2Detector", "reasoning_detector": "KimiK2Detector", "auto_detected": { "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2" }, "model": "moonshotai/Kimi-K2.6", "tokenizer": { "repo": "moonshotai/Kimi-K2.6", "revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK25ForConditionalGeneration" ], "model_type": "kimi_k25" }, "hf_config_error": null, "chat_template_sha256": "8bf859698fd4781c0e1e1c63ce74422aab27e53ccc5f47116317c64cda06132f", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 163588, 69702, 163601, 163606 ] }, "stop": { "token": "<|im_end|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k26-no-call", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2", "tool_call_detector": "KimiK2Detector", "reasoning_detector": "KimiK2Detector", "auto_detected": { "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2" }, "model": "moonshotai/Kimi-K2.6", "tokenizer": { "repo": "moonshotai/Kimi-K2.6", "revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK25ForConditionalGeneration" ], "model_type": "kimi_k25" }, "hf_config_error": null, "chat_template_sha256": "8bf859698fd4781c0e1e1c63ce74422aab27e53ccc5f47116317c64cda06132f", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 163588, 69702, 163601, 163606 ] }, "stop": { "token": "<|im_end|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k26-nested-json", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2", "tool_call_detector": "KimiK2Detector", "reasoning_detector": "KimiK2Detector", "auto_detected": { "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2" }, "model": "moonshotai/Kimi-K2.6", "tokenizer": { "repo": "moonshotai/Kimi-K2.6", "revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK25ForConditionalGeneration" ], "model_type": "kimi_k25" }, "hf_config_error": null, "chat_template_sha256": "8bf859698fd4781c0e1e1c63ce74422aab27e53ccc5f47116317c64cda06132f", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 163588, 69702, 163601, 163606 ] }, "stop": { "token": "<|im_end|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k26-unicode-emoji", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2", "tool_call_detector": "KimiK2Detector", "reasoning_detector": "KimiK2Detector", "auto_detected": { "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2" }, "model": "moonshotai/Kimi-K2.6", "tokenizer": { "repo": "moonshotai/Kimi-K2.6", "revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK25ForConditionalGeneration" ], "model_type": "kimi_k25" }, "hf_config_error": null, "chat_template_sha256": "8bf859698fd4781c0e1e1c63ce74422aab27e53ccc5f47116317c64cda06132f", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 163588, 69702, 163601, 163606 ] }, "stop": { "token": "<|im_end|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k26-empty-arguments", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2", "tool_call_detector": "KimiK2Detector", "reasoning_detector": "KimiK2Detector", "auto_detected": { "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2" }, "model": "moonshotai/Kimi-K2.6", "tokenizer": { "repo": "moonshotai/Kimi-K2.6", "revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK25ForConditionalGeneration" ], "model_type": "kimi_k25" }, "hf_config_error": null, "chat_template_sha256": "8bf859698fd4781c0e1e1c63ce74422aab27e53ccc5f47116317c64cda06132f", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 163588, 69702, 163601, 163606 ] }, "stop": { "token": "<|im_end|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k26-string-escapes", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2", "tool_call_detector": "KimiK2Detector", "reasoning_detector": "KimiK2Detector", "auto_detected": { "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2" }, "model": "moonshotai/Kimi-K2.6", "tokenizer": { "repo": "moonshotai/Kimi-K2.6", "revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK25ForConditionalGeneration" ], "model_type": "kimi_k25" }, "hf_config_error": null, "chat_template_sha256": "8bf859698fd4781c0e1e1c63ce74422aab27e53ccc5f47116317c64cda06132f", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 163588, 69702, 163601, 163606 ] }, "stop": { "token": "<|im_end|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k26-marker-in-arguments", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2", "tool_call_detector": "KimiK2Detector", "reasoning_detector": "KimiK2Detector", "auto_detected": { "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2" }, "model": "moonshotai/Kimi-K2.6", "tokenizer": { "repo": "moonshotai/Kimi-K2.6", "revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK25ForConditionalGeneration" ], "model_type": "kimi_k25" }, "hf_config_error": null, "chat_template_sha256": "8bf859698fd4781c0e1e1c63ce74422aab27e53ccc5f47116317c64cda06132f", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 163588, 69702, 163601, 163606 ] }, "stop": { "token": "<|im_end|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k26-marker-terminator-in-string", "family": "kimi", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected None, got ' please\"}'; tool_calls[0].arguments: expected '{\"path\": \"end.txt\", \"content\": \"stop at <|tool_call_end|> please\"}', got '{\"path\": \"end.txt\", \"content\": \"stop at '" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected None, got ' please\"}'; tool_calls[0].arguments: expected '{\"path\": \"end.txt\", \"content\": \"stop at <|tool_call_end|> please\"}', got '{\"path\": \"end.txt\", \"content\": \"stop at '" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected None, got ' please\"}'; tool_calls[0].arguments: expected '{\"path\": \"end.txt\", \"content\": \"stop at <|tool_call_end|> please\"}', got '{\"path\": \"end.txt\", \"content\": \"stop at '" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected None, got ' please\"}'; tool_calls[0].arguments: expected '{\"path\": \"end.txt\", \"content\": \"stop at <|tool_call_end|> please\"}', got '{\"path\": \"end.txt\", \"content\": \"stop at '" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected None, got ' please\"}'; tool_calls[0].arguments: expected '{\"path\": \"end.txt\", \"content\": \"stop at <|tool_call_end|> please\"}', got '{\"path\": \"end.txt\", \"content\": \"stop at '" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected None, got ' please\"}'; tool_calls[0].arguments: expected '{\"path\": \"end.txt\", \"content\": \"stop at <|tool_call_end|> please\"}', got '{\"path\": \"end.txt\", \"content\": \"stop at '" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected None, got ' please\"}'; tool_calls[0].arguments: expected '{\"path\": \"end.txt\", \"content\": \"stop at <|tool_call_end|> please\"}', got '{\"path\": \"end.txt\", \"content\": \"stop at '" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected None, got ' please\"}'; tool_calls[0].arguments: expected '{\"path\": \"end.txt\", \"content\": \"stop at <|tool_call_end|> please\"}', got '{\"path\": \"end.txt\", \"content\": \"stop at '" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream None, got ' please\"}'; tool_calls[0].arguments: nonstream '{\"path\": \"end.txt\", \"content\": \"stop at <|tool_call_end|> please\"}', got '{\"path\": \"end.txt\", \"content\": \"stop at '" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream None, got ' please\"}'; tool_calls[0].arguments: nonstream '{\"path\": \"end.txt\", \"content\": \"stop at <|tool_call_end|> please\"}', got '{\"path\": \"end.txt\", \"content\": \"stop at '" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream None, got ' please\"}'; tool_calls[0].arguments: nonstream '{\"path\": \"end.txt\", \"content\": \"stop at <|tool_call_end|> please\"}', got '{\"path\": \"end.txt\", \"content\": \"stop at '" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream None, got ' please\"}'; tool_calls[0].arguments: nonstream '{\"path\": \"end.txt\", \"content\": \"stop at <|tool_call_end|> please\"}', got '{\"path\": \"end.txt\", \"content\": \"stop at '" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream None, got ' please\"}'; tool_calls[0].arguments: nonstream '{\"path\": \"end.txt\", \"content\": \"stop at <|tool_call_end|> please\"}', got '{\"path\": \"end.txt\", \"content\": \"stop at '" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream None, got ' please\"}'; tool_calls[0].arguments: nonstream '{\"path\": \"end.txt\", \"content\": \"stop at <|tool_call_end|> please\"}', got '{\"path\": \"end.txt\", \"content\": \"stop at '" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream None, got ' please\"}'; tool_calls[0].arguments: nonstream '{\"path\": \"end.txt\", \"content\": \"stop at <|tool_call_end|> please\"}', got '{\"path\": \"end.txt\", \"content\": \"stop at '" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream None, got ' please\"}'; tool_calls[0].arguments: nonstream '{\"path\": \"end.txt\", \"content\": \"stop at <|tool_call_end|> please\"}', got '{\"path\": \"end.txt\", \"content\": \"stop at '" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"end.txt\", \"content\": \"stop at ': Unterminated string starting at: line 1 column 32 (char 31))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"end.txt\", \"content\": \"stop at ': Unterminated string starting at: line 1 column 32 (char 31))" }, { "check": "arguments_json", "strategy": "token", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"end.txt\", \"content\": \"stop at ': Unterminated string starting at: line 1 column 32 (char 31))" }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"end.txt\", \"content\": \"stop at ': Unterminated string starting at: line 1 column 32 (char 31))" }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"end.txt\", \"content\": \"stop at ': Unterminated string starting at: line 1 column 32 (char 31))" }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"end.txt\", \"content\": \"stop at ': Unterminated string starting at: line 1 column 32 (char 31))" }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"end.txt\", \"content\": \"stop at ': Unterminated string starting at: line 1 column 32 (char 31))" }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"end.txt\", \"content\": \"stop at ': Unterminated string starting at: line 1 column 32 (char 31))" }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2", "tool_call_detector": "KimiK2Detector", "reasoning_detector": "KimiK2Detector", "auto_detected": { "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2" }, "model": "moonshotai/Kimi-K2.6", "tokenizer": { "repo": "moonshotai/Kimi-K2.6", "revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK25ForConditionalGeneration" ], "model_type": "kimi_k25" }, "hf_config_error": null, "chat_template_sha256": "8bf859698fd4781c0e1e1c63ce74422aab27e53ccc5f47116317c64cda06132f", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 163588, 69702, 163601, 163606 ] }, "stop": { "token": "<|im_end|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "Write the literal end marker into the file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"end.txt\", \"content\": \"stop at <|tool_call_end|> please\"}" } ], "exception": null }, "streams": { "one": { "content": " please\"}", "reasoning_content": "Write the literal end marker into the file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"end.txt\", \"content\": \"stop at " } ], "exception": null }, "special": { "content": " please\"}", "reasoning_content": "Write the literal end marker into the file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"end.txt\", \"content\": \"stop at " } ], "exception": null }, "token": { "content": " please\"}", "reasoning_content": "Write the literal end marker into the file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"end.txt\", \"content\": \"stop at " } ], "exception": null }, "rand:1:8": { "content": " please\"}", "reasoning_content": "Write the literal end marker into the file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"end.txt\", \"content\": \"stop at " } ], "exception": null }, "rand:2:8": { "content": " please\"}", "reasoning_content": "Write the literal end marker into the file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"end.txt\", \"content\": \"stop at " } ], "exception": null }, "rand:3:8": { "content": " please\"}", "reasoning_content": "Write the literal end marker into the file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"end.txt\", \"content\": \"stop at " } ], "exception": null }, "rand:4:8": { "content": " please\"}", "reasoning_content": "Write the literal end marker into the file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"end.txt\", \"content\": \"stop at " } ], "exception": null }, "rand:5:8": { "content": " please\"}", "reasoning_content": "Write the literal end marker into the file.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"end.txt\", \"content\": \"stop at " } ], "exception": null } } } }, { "fixture_id": "kimi/k26-empty-think", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2", "tool_call_detector": "KimiK2Detector", "reasoning_detector": "KimiK2Detector", "auto_detected": { "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2" }, "model": "moonshotai/Kimi-K2.6", "tokenizer": { "repo": "moonshotai/Kimi-K2.6", "revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK25ForConditionalGeneration" ], "model_type": "kimi_k25" }, "hf_config_error": null, "chat_template_sha256": "8bf859698fd4781c0e1e1c63ce74422aab27e53ccc5f47116317c64cda06132f", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 163588, 69702, 163601, 163606 ] }, "stop": { "token": "<|im_end|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k26-thinking-disabled-call", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2", "tool_call_detector": "KimiK2Detector", "reasoning_detector": "KimiK2Detector", "auto_detected": { "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2" }, "model": "moonshotai/Kimi-K2.6", "tokenizer": { "repo": "moonshotai/Kimi-K2.6", "revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK25ForConditionalGeneration" ], "model_type": "kimi_k25" }, "hf_config_error": null, "chat_template_sha256": "8bf859698fd4781c0e1e1c63ce74422aab27e53ccc5f47116317c64cda06132f", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 163588, 69702, 163601, 163606, 163607 ] }, "stop": { "token": "<|im_end|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k26-multi-turn-second-call", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2", "tool_call_detector": "KimiK2Detector", "reasoning_detector": "KimiK2Detector", "auto_detected": { "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2" }, "model": "moonshotai/Kimi-K2.6", "tokenizer": { "repo": "moonshotai/Kimi-K2.6", "revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK25ForConditionalGeneration" ], "model_type": "kimi_k25" }, "hf_config_error": null, "chat_template_sha256": "8bf859698fd4781c0e1e1c63ce74422aab27e53ccc5f47116317c64cda06132f", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 163588, 69702, 163601, 163606 ] }, "stop": { "token": "<|im_end|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k2i-single-call", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k2", "reasoning_parser": null, "tool_call_detector": "KimiK2Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2" }, "model": "moonshotai/Kimi-K2-Instruct-0905", "tokenizer": { "repo": "moonshotai/Kimi-K2-Instruct-0905", "revision": "ac6c49f04883bd0a0598b790693a72061c676629", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "kimi_k2" }, "hf_config_error": null, "chat_template_sha256": "00938c355ca7195fd4f9c923e93af063846d4d5005e06999274f8db57b65bb0b", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 163588, 69702, 163601 ] }, "stop": { "token": "<|im_end|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k2i-content-before-call", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k2", "reasoning_parser": null, "tool_call_detector": "KimiK2Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2" }, "model": "moonshotai/Kimi-K2-Instruct-0905", "tokenizer": { "repo": "moonshotai/Kimi-K2-Instruct-0905", "revision": "ac6c49f04883bd0a0598b790693a72061c676629", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "kimi_k2" }, "hf_config_error": null, "chat_template_sha256": "00938c355ca7195fd4f9c923e93af063846d4d5005e06999274f8db57b65bb0b", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 163588, 69702, 163601 ] }, "stop": { "token": "<|im_end|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k2i-no-call", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k2", "reasoning_parser": null, "tool_call_detector": "KimiK2Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2" }, "model": "moonshotai/Kimi-K2-Instruct-0905", "tokenizer": { "repo": "moonshotai/Kimi-K2-Instruct-0905", "revision": "ac6c49f04883bd0a0598b790693a72061c676629", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "DeepseekV3ForCausalLM" ], "model_type": "kimi_k2" }, "hf_config_error": null, "chat_template_sha256": "00938c355ca7195fd4f9c923e93af063846d4d5005e06999274f8db57b65bb0b", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 163588, 69702, 163601 ] }, "stop": { "token": "<|im_end|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k3-single-call", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k3", "reasoning_parser": "kimi_k3", "tool_call_detector": "KimiK3Detector", "reasoning_detector": "KimiK3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "moonshotai/Kimi-K3", "tokenizer": { "repo": "moonshotai/Kimi-K3", "revision": "f831ab66814297da540d832a5235f8e904f29d06", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK3ForConditionalGeneration" ], "model_type": "kimi_k3" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "kimi_k3", "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1, 163589, 163587, 39964, 163589 ] }, "stop": { "token": "<|end_of_msg|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k3-parallel-same-tool", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k3", "reasoning_parser": "kimi_k3", "tool_call_detector": "KimiK3Detector", "reasoning_detector": "KimiK3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "moonshotai/Kimi-K3", "tokenizer": { "repo": "moonshotai/Kimi-K3", "revision": "f831ab66814297da540d832a5235f8e904f29d06", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK3ForConditionalGeneration" ], "model_type": "kimi_k3" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "kimi_k3", "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1, 163589, 163587, 39964, 163589 ] }, "stop": { "token": "<|end_of_msg|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k3-content-before-call", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k3", "reasoning_parser": "kimi_k3", "tool_call_detector": "KimiK3Detector", "reasoning_detector": "KimiK3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "moonshotai/Kimi-K3", "tokenizer": { "repo": "moonshotai/Kimi-K3", "revision": "f831ab66814297da540d832a5235f8e904f29d06", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK3ForConditionalGeneration" ], "model_type": "kimi_k3" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "kimi_k3", "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1, 163589, 163587, 39964, 163589 ] }, "stop": { "token": "<|end_of_msg|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k3-no-call", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k3", "reasoning_parser": "kimi_k3", "tool_call_detector": "KimiK3Detector", "reasoning_detector": "KimiK3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "moonshotai/Kimi-K3", "tokenizer": { "repo": "moonshotai/Kimi-K3", "revision": "f831ab66814297da540d832a5235f8e904f29d06", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK3ForConditionalGeneration" ], "model_type": "kimi_k3" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "kimi_k3", "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1, 163589, 163587, 39964, 163589 ] }, "stop": { "token": "<|end_of_msg|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k3-nested-json", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k3", "reasoning_parser": "kimi_k3", "tool_call_detector": "KimiK3Detector", "reasoning_detector": "KimiK3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "moonshotai/Kimi-K3", "tokenizer": { "repo": "moonshotai/Kimi-K3", "revision": "f831ab66814297da540d832a5235f8e904f29d06", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK3ForConditionalGeneration" ], "model_type": "kimi_k3" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "kimi_k3", "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1, 163589, 163587, 39964, 163589 ] }, "stop": { "token": "<|end_of_msg|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k3-typed-values", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k3", "reasoning_parser": "kimi_k3", "tool_call_detector": "KimiK3Detector", "reasoning_detector": "KimiK3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "moonshotai/Kimi-K3", "tokenizer": { "repo": "moonshotai/Kimi-K3", "revision": "f831ab66814297da540d832a5235f8e904f29d06", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK3ForConditionalGeneration" ], "model_type": "kimi_k3" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "kimi_k3", "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1, 163589, 163587, 39964, 163589 ] }, "stop": { "token": "<|end_of_msg|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k3-empty-arguments", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k3", "reasoning_parser": "kimi_k3", "tool_call_detector": "KimiK3Detector", "reasoning_detector": "KimiK3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "moonshotai/Kimi-K3", "tokenizer": { "repo": "moonshotai/Kimi-K3", "revision": "f831ab66814297da540d832a5235f8e904f29d06", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK3ForConditionalGeneration" ], "model_type": "kimi_k3" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "kimi_k3", "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1, 163589, 163587, 39964, 163589 ] }, "stop": { "token": "<|end_of_msg|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k3-string-escapes", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k3", "reasoning_parser": "kimi_k3", "tool_call_detector": "KimiK3Detector", "reasoning_detector": "KimiK3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "moonshotai/Kimi-K3", "tokenizer": { "repo": "moonshotai/Kimi-K3", "revision": "f831ab66814297da540d832a5235f8e904f29d06", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK3ForConditionalGeneration" ], "model_type": "kimi_k3" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "kimi_k3", "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1, 163589, 163587, 39964, 163589 ] }, "stop": { "token": "<|end_of_msg|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k3-control-marker-text-in-value", "family": "kimi", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"x.txt\", \"content\": \"<|close|>argument<|sep|> is how args end\"}', got '{\"path\": \"x.txt\", \"content\": \"\"}'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"x.txt\", \"content\": \"<|close|>argument<|sep|> is how args end\"}', got '{\"path\": \"x.txt\", \"content\": \"\"}'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"x.txt\", \"content\": \"<|close|>argument<|sep|> is how args end\"}', got '{\"path\": \"x.txt\", \"content\": \"\"}'" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"x.txt\", \"content\": \"<|close|>argument<|sep|> is how args end\"}', got '{\"path\": \"x.txt\", \"content\": \"\"}'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"x.txt\", \"content\": \"<|close|>argument<|sep|> is how args end\"}', got '{\"path\": \"x.txt\", \"content\": \"\"}'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"x.txt\", \"content\": \"<|close|>argument<|sep|> is how args end\"}', got '{\"path\": \"x.txt\", \"content\": \"\"}'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"x.txt\", \"content\": \"<|close|>argument<|sep|> is how args end\"}', got '{\"path\": \"x.txt\", \"content\": \"\"}'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"x.txt\", \"content\": \"<|close|>argument<|sep|> is how args end\"}', got '{\"path\": \"x.txt\", \"content\": \"\"}'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"x.txt\", \"content\": \"<|close|>argument<|sep|> is how args end\"}', got '{\"path\": \"x.txt\", \"content\": \"\"}'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k3", "reasoning_parser": "kimi_k3", "tool_call_detector": "KimiK3Detector", "reasoning_detector": "KimiK3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "moonshotai/Kimi-K3", "tokenizer": { "repo": "moonshotai/Kimi-K3", "revision": "f831ab66814297da540d832a5235f8e904f29d06", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK3ForConditionalGeneration" ], "model_type": "kimi_k3" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "kimi_k3", "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1, 163589, 163587, 39964, 163589 ] }, "stop": { "token": "<|end_of_msg|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "Write the literal XTML marker text.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"x.txt\", \"content\": \"\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "Write the literal XTML marker text.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"x.txt\", \"content\": \"\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "Write the literal XTML marker text.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"x.txt\", \"content\": \"\"}" } ], "exception": null }, "token": { "content": null, "reasoning_content": "Write the literal XTML marker text.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"x.txt\", \"content\": \"\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "Write the literal XTML marker text.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"x.txt\", \"content\": \"\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "Write the literal XTML marker text.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"x.txt\", \"content\": \"\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "Write the literal XTML marker text.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"x.txt\", \"content\": \"\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "Write the literal XTML marker text.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"x.txt\", \"content\": \"\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "Write the literal XTML marker text.", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"x.txt\", \"content\": \"\"}" } ], "exception": null } } } }, { "fixture_id": "kimi/k3-xtml-attr-escaping", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k3", "reasoning_parser": "kimi_k3", "tool_call_detector": "KimiK3Detector", "reasoning_detector": "KimiK3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "moonshotai/Kimi-K3", "tokenizer": { "repo": "moonshotai/Kimi-K3", "revision": "f831ab66814297da540d832a5235f8e904f29d06", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK3ForConditionalGeneration" ], "model_type": "kimi_k3" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "kimi_k3", "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1, 163589, 163587, 39964, 163589 ] }, "stop": { "token": "<|end_of_msg|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k3-long-arguments", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k3", "reasoning_parser": "kimi_k3", "tool_call_detector": "KimiK3Detector", "reasoning_detector": "KimiK3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "moonshotai/Kimi-K3", "tokenizer": { "repo": "moonshotai/Kimi-K3", "revision": "f831ab66814297da540d832a5235f8e904f29d06", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK3ForConditionalGeneration" ], "model_type": "kimi_k3" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "kimi_k3", "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1, 163589, 163587, 39964, 163589 ] }, "stop": { "token": "<|end_of_msg|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k3-empty-think", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k3", "reasoning_parser": "kimi_k3", "tool_call_detector": "KimiK3Detector", "reasoning_detector": "KimiK3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "moonshotai/Kimi-K3", "tokenizer": { "repo": "moonshotai/Kimi-K3", "revision": "f831ab66814297da540d832a5235f8e904f29d06", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK3ForConditionalGeneration" ], "model_type": "kimi_k3" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "kimi_k3", "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1, 163589, 163587, 39964, 163589 ] }, "stop": { "token": "<|end_of_msg|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k3-thinking-disabled-call", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k3", "reasoning_parser": "kimi_k3", "tool_call_detector": "KimiK3Detector", "reasoning_detector": "KimiK3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "moonshotai/Kimi-K3", "tokenizer": { "repo": "moonshotai/Kimi-K3", "revision": "f831ab66814297da540d832a5235f8e904f29d06", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK3ForConditionalGeneration" ], "model_type": "kimi_k3" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "kimi_k3", "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1, 163589, 163587, 12092, 163589 ] }, "stop": { "token": "<|end_of_msg|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k3-thinking-disabled-no-call", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k3", "reasoning_parser": "kimi_k3", "tool_call_detector": "KimiK3Detector", "reasoning_detector": "KimiK3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "moonshotai/Kimi-K3", "tokenizer": { "repo": "moonshotai/Kimi-K3", "revision": "f831ab66814297da540d832a5235f8e904f29d06", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK3ForConditionalGeneration" ], "model_type": "kimi_k3" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "kimi_k3", "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1, 163589, 163587, 12092, 163589 ] }, "stop": { "token": "<|end_of_msg|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k3-multi-turn-second-call", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k3", "reasoning_parser": "kimi_k3", "tool_call_detector": "KimiK3Detector", "reasoning_detector": "KimiK3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "moonshotai/Kimi-K3", "tokenizer": { "repo": "moonshotai/Kimi-K3", "revision": "f831ab66814297da540d832a5235f8e904f29d06", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK3ForConditionalGeneration" ], "model_type": "kimi_k3" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "kimi_k3", "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1, 163589, 163587, 39964, 163589 ] }, "stop": { "token": "<|end_of_msg|>", "id": 163586, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k26-truncated-inside-arguments", "family": "kimi", "status": "fail", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the argument JSON; the call never closes." }, { "check": "expected_error", "strategy": "special", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the argument JSON; the call never closes." }, { "check": "expected_error", "strategy": "token", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the argument JSON; the call never closes." }, { "check": "expected_error", "strategy": "rand:1:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the argument JSON; the call never closes." }, { "check": "expected_error", "strategy": "rand:2:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the argument JSON; the call never closes." }, { "check": "expected_error", "strategy": "rand:3:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the argument JSON; the call never closes." }, { "check": "expected_error", "strategy": "rand:4:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the argument JSON; the call never closes." }, { "check": "expected_error", "strategy": "rand:5:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for max_tokens hit inside the argument JSON; the call never closes." }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Madrid\", \"unit\":': Expecting value: line 1 column 27 (char 26))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Madrid\", \"unit\":': Expecting value: line 1 column 27 (char 26))" }, { "check": "arguments_json", "strategy": "token", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Madrid\", \"unit\":': Expecting value: line 1 column 27 (char 26))" }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Madrid\", \"unit\":': Expecting value: line 1 column 27 (char 26))" }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Madrid\", \"unit\":': Expecting value: line 1 column 27 (char 26))" }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Madrid\", \"unit\":': Expecting value: line 1 column 27 (char 26))" }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Madrid\", \"unit\":': Expecting value: line 1 column 27 (char 26))" }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('{\"city\": \"Madrid\", \"unit\":': Expecting value: line 1 column 27 (char 26))" }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2", "tool_call_detector": "KimiK2Detector", "reasoning_detector": "KimiK2Detector", "auto_detected": { "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2" }, "model": "moonshotai/Kimi-K2.6", "tokenizer": { "repo": "moonshotai/Kimi-K2.6", "revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK25ForConditionalGeneration" ], "model_type": "kimi_k25" }, "hf_config_error": null, "chat_template_sha256": "8bf859698fd4781c0e1e1c63ce74422aab27e53ccc5f47116317c64cda06132f", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 163588, 69702, 163601, 163606, 163607 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Madrid\", \"unit\":" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Madrid\", \"unit\":" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Madrid\", \"unit\":" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Madrid\", \"unit\":" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Madrid\", \"unit\":" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Madrid\", \"unit\":" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Madrid\", \"unit\":" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Madrid\", \"unit\":" } ], "exception": null } } } }, { "fixture_id": "kimi/k3-truncated-inside-argument", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k3", "reasoning_parser": "kimi_k3", "tool_call_detector": "KimiK3Detector", "reasoning_detector": "KimiK3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "moonshotai/Kimi-K3", "tokenizer": { "repo": "moonshotai/Kimi-K3", "revision": "f831ab66814297da540d832a5235f8e904f29d06", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK3ForConditionalGeneration" ], "model_type": "kimi_k3" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "kimi_k3", "thinking": false, "chat_template_kwargs": { "thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1, 163589, 163587, 12092, 163589 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k26-truncated-in-reasoning", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2", "tool_call_detector": "KimiK2Detector", "reasoning_detector": "KimiK2Detector", "auto_detected": { "tool_call_parser": "kimi_k2", "reasoning_parser": "kimi_k2" }, "model": "moonshotai/Kimi-K2.6", "tokenizer": { "repo": "moonshotai/Kimi-K2.6", "revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK25ForConditionalGeneration" ], "model_type": "kimi_k25" }, "hf_config_error": null, "chat_template_sha256": "8bf859698fd4781c0e1e1c63ce74422aab27e53ccc5f47116317c64cda06132f", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 163588, 69702, 163601, 163606 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "kimi/k3-truncated-in-reasoning", "family": "kimi", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "kimi_k3", "reasoning_parser": "kimi_k3", "tool_call_detector": "KimiK3Detector", "reasoning_detector": "KimiK3Detector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "moonshotai/Kimi-K3", "tokenizer": { "repo": "moonshotai/Kimi-K3", "revision": "f831ab66814297da540d832a5235f8e904f29d06", "class": "TikTokenTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": true }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "KimiK3ForConditionalGeneration" ], "model_type": "kimi_k3" }, "hf_config_error": null, "chat_template_sha256": null, "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": "kimi_k3", "thinking": true, "chat_template_kwargs": { "thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1, 163589, 163587, 39964, 163589 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "llama/l3-vllm-parameters-key", "family": "llama", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"query\": \"test query\", \"limit\": 10}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"query\": \"test query\", \"limit\": 10}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"query\": \"test query\", \"limit\": 10}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"query\": \"test query\", \"limit\": 10}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{\"query\": \"test query\", \"limit\": 10}' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"query\": \"test query\", \"limit\": 10}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{\"query\": \"test query\", \"limit\": 10}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"query\": \"test query\", \"limit\": 10}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"query\": \"test query\", \"limit\": 10}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"query\": \"test query\", \"limit\": 10}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] searchTool: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] searchTool: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] searchTool: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] searchTool: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "llama3", "reasoning_parser": null, "tool_call_detector": "Llama32Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-3.3-70B-Instruct", "tokenizer": { "repo": "unsloth/Llama-3.3-70B-Instruct", "revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128006, 78191, 128007, 271 ] }, "stop": { "token": "<|end_of_text|>", "id": 128001, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "searchTool", "arguments_raw": "{\"query\": \"test query\", \"limit\": 10}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "searchTool", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "searchTool", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "searchTool", "arguments_raw": "{\"query\": \"test query\", \"limit\": 10}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "searchTool", "arguments_raw": "{\"query\": \"test query\", \"limit\": 10}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "searchTool", "arguments_raw": "{\"query\": \"test query\", \"limit\": 10}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "searchTool", "arguments_raw": "{\"query\": \"test query\", \"limit\": 10}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "searchTool", "arguments_raw": "{\"query\": \"test query\", \"limit\": 10}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "searchTool", "arguments_raw": "{\"query\": \"test query\", \"limit\": 10}" } ], "exception": null } } } }, { "fixture_id": "llama/l3-vllm-arguments-key", "family": "llama", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"query\": \"test\"}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"query\": \"test\"}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"query\": \"test\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"query\": \"test\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{\"query\": \"test\"}' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"query\": \"test\"}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{\"query\": \"test\"}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"query\": \"test\"}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"query\": \"test\"}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"query\": \"test\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] searchTool: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] searchTool: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] searchTool: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] searchTool: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "llama3", "reasoning_parser": null, "tool_call_detector": "Llama32Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-3.3-70B-Instruct", "tokenizer": { "repo": "unsloth/Llama-3.3-70B-Instruct", "revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128006, 78191, 128007, 271 ] }, "stop": { "token": "<|end_of_text|>", "id": 128001, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "searchTool", "arguments_raw": "{\"query\": \"test\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "searchTool", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "searchTool", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "searchTool", "arguments_raw": "{\"query\": \"test\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "searchTool", "arguments_raw": "{\"query\": \"test\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "searchTool", "arguments_raw": "{\"query\": \"test\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "searchTool", "arguments_raw": "{\"query\": \"test\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "searchTool", "arguments_raw": "{\"query\": \"test\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "searchTool", "arguments_raw": "{\"query\": \"test\"}" } ], "exception": null } } } }, { "fixture_id": "llama/l3-vllm-deeply-nested", "family": "llama", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"level1\": {\"level2\": {\"level3\": {\"level4\": {\"value\": \"deep\"}}}}}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"level1\": {\"level2\": {\"level3\": {\"level4\": {\"value\": \"deep\"}}}}}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"level1\": {\"level2\": {\"level3\": {\"level4\": {\"value\": \"deep\"}}}}}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"level1\": {\"level2\": {\"level3\": {\"level4\": {\"value\": \"deep\"}}}}}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{\"level1\": {\"level2\": {\"level3\": {\"level4\": {\"value\": \"deep\"}}}}}' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"level1\": {\"level2\": {\"level3\": {\"level4\": {\"value\": \"deep\"}}}}}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{\"level1\": {\"level2\": {\"level3\": {\"level4\": {\"value\": \"deep\"}}}}}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"level1\": {\"level2\": {\"level3\": {\"level4\": {\"value\": \"deep\"}}}}}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"level1\": {\"level2\": {\"level3\": {\"level4\": {\"value\": \"deep\"}}}}}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"level1\": {\"level2\": {\"level3\": {\"level4\": {\"value\": \"deep\"}}}}}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] complexTool: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] complexTool: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] complexTool: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] complexTool: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "llama3", "reasoning_parser": null, "tool_call_detector": "Llama32Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-3.3-70B-Instruct", "tokenizer": { "repo": "unsloth/Llama-3.3-70B-Instruct", "revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128006, 78191, 128007, 271 ] }, "stop": { "token": "<|end_of_text|>", "id": 128001, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "complexTool", "arguments_raw": "{\"level1\": {\"level2\": {\"level3\": {\"level4\": {\"value\": \"deep\"}}}}}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "complexTool", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "complexTool", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "complexTool", "arguments_raw": "{\"level1\": {\"level2\": {\"level3\": {\"level4\": {\"value\": \"deep\"}}}}}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "complexTool", "arguments_raw": "{\"level1\": {\"level2\": {\"level3\": {\"level4\": {\"value\": \"deep\"}}}}}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "complexTool", "arguments_raw": "{\"level1\": {\"level2\": {\"level3\": {\"level4\": {\"value\": \"deep\"}}}}}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "complexTool", "arguments_raw": "{\"level1\": {\"level2\": {\"level3\": {\"level4\": {\"value\": \"deep\"}}}}}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "complexTool", "arguments_raw": "{\"level1\": {\"level2\": {\"level3\": {\"level4\": {\"value\": \"deep\"}}}}}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "complexTool", "arguments_raw": "{\"level1\": {\"level2\": {\"level3\": {\"level4\": {\"value\": \"deep\"}}}}}" } ], "exception": null } } } }, { "fixture_id": "llama/l3-vllm-brackets-in-strings", "family": "llama", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"query\": \"test {value} [complex]\", \"nested\": {\"inner\": \"more {brackets}\"}}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"query\": \"test {value} [complex]\", \"nested\": {\"inner\": \"more {brackets}\"}}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"query\": \"test {value} [complex]\", \"nested\": {\"inner\": \"more {brackets}\"}}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"query\": \"test {value} [complex]\", \"nested\": {\"inner\": \"more {brackets}\"}}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{\"query\": \"test {value} [complex]\", \"nested\": {\"inner\": \"more {brackets}\"}}' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"query\": \"test {value} [complex]\", \"nested\": {\"inner\": \"more {brackets}\"}}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{\"query\": \"test {value} [complex]\", \"nested\": {\"inner\": \"more {brackets}\"}}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"query\": \"test {value} [complex]\", \"nested\": {\"inner\": \"more {brackets}\"}}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"query\": \"test {value} [complex]\", \"nested\": {\"inner\": \"more {brackets}\"}}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"query\": \"test {value} [complex]\", \"nested\": {\"inner\": \"more {brackets}\"}}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] searchTool: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] searchTool: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] searchTool: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] searchTool: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "llama3", "reasoning_parser": null, "tool_call_detector": "Llama32Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-3.3-70B-Instruct", "tokenizer": { "repo": "unsloth/Llama-3.3-70B-Instruct", "revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128006, 78191, 128007, 271 ] }, "stop": { "token": "<|end_of_text|>", "id": 128001, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "searchTool", "arguments_raw": "{\"query\": \"test {value} [complex]\", \"nested\": {\"inner\": \"more {brackets}\"}}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "searchTool", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "searchTool", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "searchTool", "arguments_raw": "{\"query\": \"test {value} [complex]\", \"nested\": {\"inner\": \"more {brackets}\"}}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "searchTool", "arguments_raw": "{\"query\": \"test {value} [complex]\", \"nested\": {\"inner\": \"more {brackets}\"}}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "searchTool", "arguments_raw": "{\"query\": \"test {value} [complex]\", \"nested\": {\"inner\": \"more {brackets}\"}}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "searchTool", "arguments_raw": "{\"query\": \"test {value} [complex]\", \"nested\": {\"inner\": \"more {brackets}\"}}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "searchTool", "arguments_raw": "{\"query\": \"test {value} [complex]\", \"nested\": {\"inner\": \"more {brackets}\"}}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "searchTool", "arguments_raw": "{\"query\": \"test {value} [complex]\", \"nested\": {\"inner\": \"more {brackets}\"}}" } ], "exception": null } } } }, { "fixture_id": "llama/l3-vllm-escaped-quotes", "family": "llama", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"text\": \"He said \\\\\"Hello {world}\\\\\"\"}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"text\": \"He said \\\\\"Hello {world}\\\\\"\"}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"He said \\\\\"Hello {world}\\\\\"\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"He said \\\\\"Hello {world}\\\\\"\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{\"text\": \"He said \\\\\"Hello {world}\\\\\"\"}' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"text\": \"He said \\\\\"Hello {world}\\\\\"\"}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{\"text\": \"He said \\\\\"Hello {world}\\\\\"\"}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"text\": \"He said \\\\\"Hello {world}\\\\\"\"}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"text\": \"He said \\\\\"Hello {world}\\\\\"\"}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"text\": \"He said \\\\\"Hello {world}\\\\\"\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] parserTool: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] parserTool: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] parserTool: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] parserTool: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "llama3", "reasoning_parser": null, "tool_call_detector": "Llama32Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-3.3-70B-Instruct", "tokenizer": { "repo": "unsloth/Llama-3.3-70B-Instruct", "revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128006, 78191, 128007, 271 ] }, "stop": { "token": "<|end_of_text|>", "id": 128001, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "parserTool", "arguments_raw": "{\"text\": \"He said \\\"Hello {world}\\\"\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "parserTool", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "parserTool", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "parserTool", "arguments_raw": "{\"text\": \"He said \\\"Hello {world}\\\"\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "parserTool", "arguments_raw": "{\"text\": \"He said \\\"Hello {world}\\\"\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "parserTool", "arguments_raw": "{\"text\": \"He said \\\"Hello {world}\\\"\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "parserTool", "arguments_raw": "{\"text\": \"He said \\\"Hello {world}\\\"\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "parserTool", "arguments_raw": "{\"text\": \"He said \\\"Hello {world}\\\"\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "parserTool", "arguments_raw": "{\"text\": \"He said \\\"Hello {world}\\\"\"}" } ], "exception": null } } } }, { "fixture_id": "llama/l3-vllm-json-without-name-is-content", "family": "llama", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected '{\"parameters\": {}}', got None" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected '{\"parameters\": {}}', got None" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected '{\"parameters\": {}}', got None" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected '{\"parameters\": {}}', got None" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected '{\"parameters\": {}}', got None" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected '{\"parameters\": {}}', got None" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected '{\"parameters\": {}}', got None" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected '{\"parameters\": {}}', got None" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected '{\"parameters\": {}}', got None" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "llama3", "reasoning_parser": null, "tool_call_detector": "Llama32Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-3.3-70B-Instruct", "tokenizer": { "repo": "unsloth/Llama-3.3-70B-Instruct", "revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128006, 78191, 128007, 271 ] }, "stop": { "token": "<|end_of_text|>", "id": 128001, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null } } } }, { "fixture_id": "llama/l3-sglang-python-tag-call", "family": "llama", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Beijing\"}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Beijing\"}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Beijing\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Beijing\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Beijing\"}' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Beijing\"}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Beijing\"}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Beijing\"}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Beijing\"}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Beijing\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "llama3", "reasoning_parser": null, "tool_call_detector": "Llama32Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-3.3-70B-Instruct", "tokenizer": { "repo": "unsloth/Llama-3.3-70B-Instruct", "revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128006, 78191, 128007, 271 ] }, "stop": { "token": "<|end_of_text|>", "id": 128001, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Beijing\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Beijing\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Beijing\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Beijing\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Beijing\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Beijing\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Beijing\"}" } ], "exception": null } } } }, { "fixture_id": "llama/l3-sglang-text-before-python-tag", "family": "llama", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected 'Let me check. ', got None; tool_calls[0].arguments: expected '{\"city\": \"Tokyo\"}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Tokyo\"}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check. ', got 'Let me check.'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check. ', got 'Let me check.'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream 'Let me check. ', got None; tool_calls[0].arguments: nonstream '{\"city\": \"Tokyo\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Tokyo\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Let me check. ', got 'Let me check.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Let me check. ', got 'Let me check.'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: special (fail): content: one None, got 'Let me check. ' | token (fail): content: one None, got 'Let me check. '; tool_calls[0].arguments: one '', got '{\"city\": \"Tokyo\"}' | rand:1:8 (fail): content: one None, got 'Let me check. '; tool_calls[0].arguments: one '', got '{\"city\": \"Tokyo\"}' | rand:2:8 (fail): content: one None, got 'Let me check. '; tool_calls[0].arguments: one '', got '{\"city\": \"Tokyo\"}' | rand:3:8 (fail): content: one None, got 'Let me check.'; tool_calls[0].arguments: one '', got '{\"city\": \"Tokyo\"}' | rand:4:8 (fail): content: one None, got 'Let me check.'; tool_calls[0].arguments: one '', got '{\"city\": \"Tokyo\"}' | rand:5:8 (fail): content: one None, got 'Let me check. '; tool_calls[0].arguments: one '', got '{\"city\": \"Tokyo\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "llama3", "reasoning_parser": null, "tool_call_detector": "Llama32Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-3.3-70B-Instruct", "tokenizer": { "repo": "unsloth/Llama-3.3-70B-Instruct", "revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128006, 78191, 128007, 271 ] }, "stop": { "token": "<|end_of_text|>", "id": 128001, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "Let me check. ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "special": { "content": "Let me check. ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "token": { "content": "Let me check. ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null }, "rand:1:8": { "content": "Let me check. ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null }, "rand:2:8": { "content": "Let me check. ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null }, "rand:3:8": { "content": "Let me check.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null }, "rand:4:8": { "content": "Let me check.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null }, "rand:5:8": { "content": "Let me check. ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null } } } }, { "fixture_id": "llama/l4-vllm-pythonic-no-call", "family": "llama", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "pythonic", "reasoning_parser": null, "tool_call_detector": "PythonicDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "tokenizer": { "repo": "unsloth/Llama-4-Scout-17B-16E-Instruct", "revision": "afd8e498c87bda51c7ea8ec68ea2f7c066e6340b", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "93c6a232af67e2b3bc9218179ec3fea5176ec88b2a4a4fb66e51feb2ef63c0bd", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200005, 140680, 200006, 368 ] }, "stop": { "token": "<|end_of_text|>", "id": 200001, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "llama/l4-vllm-pythonic-more-types", "family": "llama", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "pythonic", "reasoning_parser": null, "tool_call_detector": "PythonicDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "tokenizer": { "repo": "unsloth/Llama-4-Scout-17B-16E-Instruct", "revision": "afd8e498c87bda51c7ea8ec68ea2f7c066e6340b", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "93c6a232af67e2b3bc9218179ec3fea5176ec88b2a4a4fb66e51feb2ef63c0bd", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200005, 140680, 200006, 368 ] }, "stop": { "token": "<|end_of_text|>", "id": 200001, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "llama/l4-vllm-pythonic-parameterless", "family": "llama", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "pythonic", "reasoning_parser": null, "tool_call_detector": "PythonicDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "tokenizer": { "repo": "unsloth/Llama-4-Scout-17B-16E-Instruct", "revision": "afd8e498c87bda51c7ea8ec68ea2f7c066e6340b", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "93c6a232af67e2b3bc9218179ec3fea5176ec88b2a4a4fb66e51feb2ef63c0bd", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200005, 140680, 200006, 368 ] }, "stop": { "token": "<|end_of_text|>", "id": 200001, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "llama/l4-vllm-pythonic-escaped-strings", "family": "llama", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "pythonic", "reasoning_parser": null, "tool_call_detector": "PythonicDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "tokenizer": { "repo": "unsloth/Llama-4-Scout-17B-16E-Instruct", "revision": "afd8e498c87bda51c7ea8ec68ea2f7c066e6340b", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "93c6a232af67e2b3bc9218179ec3fea5176ec88b2a4a4fb66e51feb2ef63c0bd", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200005, 140680, 200006, 368 ] }, "stop": { "token": "<|end_of_text|>", "id": 200001, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "llama/l4-vllm-pythonic-parallel-no-spaces", "family": "llama", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "pythonic", "reasoning_parser": null, "tool_call_detector": "PythonicDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "tokenizer": { "repo": "unsloth/Llama-4-Scout-17B-16E-Instruct", "revision": "afd8e498c87bda51c7ea8ec68ea2f7c066e6340b", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "93c6a232af67e2b3bc9218179ec3fea5176ec88b2a4a4fb66e51feb2ef63c0bd", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200005, 140680, 200006, 368 ] }, "stop": { "token": "<|end_of_text|>", "id": 200001, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "llama/l4-vllm-pythonic-python-start-end", "family": "llama", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "pythonic", "reasoning_parser": null, "tool_call_detector": "PythonicDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "tokenizer": { "repo": "unsloth/Llama-4-Scout-17B-16E-Instruct", "revision": "afd8e498c87bda51c7ea8ec68ea2f7c066e6340b", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "93c6a232af67e2b3bc9218179ec3fea5176ec88b2a4a4fb66e51feb2ef63c0bd", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200005, 140680, 200006, 368 ] }, "stop": { "token": "<|end_of_text|>", "id": 200001, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "llama/l3-bug-leading-json-object-in-content", "family": "llama", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected '{\"a\": 1} is a dict', got 'is a dict'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected '{\"a\": 1} is a dict', got None" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected '{\"a\": 1} is a dict', got None" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected '{\"a\": 1} is a dict', got None" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected '{\"a\": 1} is a dict', got None" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected '{\"a\": 1} is a dict', got None" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected '{\"a\": 1} is a dict', got None" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected '{\"a\": 1} is a dict', got None" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected '{\"a\": 1} is a dict', got None" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream 'is a dict', got None" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream 'is a dict', got None" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream 'is a dict', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream 'is a dict', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream 'is a dict', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream 'is a dict', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream 'is a dict', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream 'is a dict', got None" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "llama3", "reasoning_parser": null, "tool_call_detector": "Llama32Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-3.3-70B-Instruct", "tokenizer": { "repo": "unsloth/Llama-3.3-70B-Instruct", "revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128006, 78191, 128007, 271 ] }, "stop": { "token": "<|end_of_text|>", "id": 128001, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "is a dict", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null } } } }, { "fixture_id": "llama/l3-bug-empty-json-object-in-content", "family": "llama", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected '{}', got None" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected '{}', got None" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected '{}', got None" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected '{}', got None" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected '{}', got None" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected '{}', got None" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected '{}', got None" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected '{}', got None" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected '{}', got None" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "llama3", "reasoning_parser": null, "tool_call_detector": "Llama32Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-3.3-70B-Instruct", "tokenizer": { "repo": "unsloth/Llama-3.3-70B-Instruct", "revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128006, 78191, 128007, 271 ] }, "stop": { "token": "<|end_of_text|>", "id": 128001, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null } } } }, { "fixture_id": "llama/l3-bug-whole-call-single-delta", "family": "llama", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Tokyo\"}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Tokyo\"}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Tokyo\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Tokyo\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Tokyo\"}' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Tokyo\"}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Tokyo\"}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Tokyo\"}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Tokyo\"}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Tokyo\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "llama3", "reasoning_parser": null, "tool_call_detector": "Llama32Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-3.3-70B-Instruct", "tokenizer": { "repo": "unsloth/Llama-3.3-70B-Instruct", "revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128006, 78191, 128007, 271 ] }, "stop": { "token": "<|end_of_text|>", "id": 128001, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\"}" } ], "exception": null } } } }, { "fixture_id": "llama/l4-bug-leading-underscore-identifier", "family": "llama", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '[search(_limit=5)]'; tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected None, got '[search(_limit=5)]'; tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected None, got '[search(_limit=5)]'; tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected None, got '[search(_limit=5)]'; tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected None, got '[search(_limit=5)]'; tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected None, got '[search(_limit=5)]'; tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected None, got '[search(_limit=5)]'; tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected None, got '[search(_limit=5)]'; tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected None, got '[search(_limit=5)]'; tool_calls: expected ['search'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "pythonic", "reasoning_parser": null, "tool_call_detector": "PythonicDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "tokenizer": { "repo": "unsloth/Llama-4-Scout-17B-16E-Instruct", "revision": "afd8e498c87bda51c7ea8ec68ea2f7c066e6340b", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "93c6a232af67e2b3bc9218179ec3fea5176ec88b2a4a4fb66e51feb2ef63c0bd", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200005, 140680, 200006, 368 ] }, "stop": { "token": "<|end_of_text|>", "id": 200001, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "[search(_limit=5)]", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": "[search(_limit=5)]", "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": "[search(_limit=5)]", "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": "[search(_limit=5)]", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:1:8": { "content": "[search(_limit=5)]", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:2:8": { "content": "[search(_limit=5)]", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:3:8": { "content": "[search(_limit=5)]", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:4:8": { "content": "[search(_limit=5)]", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:5:8": { "content": "[search(_limit=5)]", "reasoning_content": null, "tool_calls": [], "exception": null } } } }, { "fixture_id": "llama/l3-single-call", "family": "llama", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Paris\", \"unit\": \"celsius\"}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Paris\", \"unit\": \"celsius\"}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Paris\", \"unit\": \"celsius\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Paris\", \"unit\": \"celsius\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"celsius\"}' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"celsius\"}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"celsius\"}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"celsius\"}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"celsius\"}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"celsius\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "llama3", "reasoning_parser": null, "tool_call_detector": "Llama32Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-3.3-70B-Instruct", "tokenizer": { "repo": "unsloth/Llama-3.3-70B-Instruct", "revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128006, 78191, 128007, 271 ] }, "stop": { "token": "<|end_of_text|>", "id": 128001, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"celsius\"}" } ], "exception": null } } } }, { "fixture_id": "llama/l3-nested-json", "family": "llama", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] update_settings: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] update_settings: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] update_settings: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] update_settings: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "llama3", "reasoning_parser": null, "tool_call_detector": "Llama32Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-3.3-70B-Instruct", "tokenizer": { "repo": "unsloth/Llama-3.3-70B-Instruct", "revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128006, 78191, 128007, 271 ] }, "stop": { "token": "<|end_of_text|>", "id": 128001, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "update_settings", "arguments_raw": "{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "update_settings", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "update_settings", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "update_settings", "arguments_raw": "{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "update_settings", "arguments_raw": "{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "update_settings", "arguments_raw": "{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "update_settings", "arguments_raw": "{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "update_settings", "arguments_raw": "{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "update_settings", "arguments_raw": "{\"settings\": {\"theme\": {\"mode\": \"dark\", \"accent\": [255, 128, 0]}, \"notifications\": {\"email\": true, \"sms\": false}}, \"dry_run\": false}" } ], "exception": null } } } }, { "fixture_id": "llama/l3-unicode-emoji", "family": "llama", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] search_web: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] search_web: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] search_web: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] search_web: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "llama3", "reasoning_parser": null, "tool_call_detector": "Llama32Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-3.3-70B-Instruct", "tokenizer": { "repo": "unsloth/Llama-3.3-70B-Instruct", "revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128006, 78191, 128007, 271 ] }, "stop": { "token": "<|end_of_text|>", "id": 128001, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "search_web", "arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "search_web", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "search_web", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "search_web", "arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "search_web", "arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "search_web", "arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "search_web", "arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "search_web", "arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "search_web", "arguments_raw": "{\"query\": \"Zürich café ☕ 東京の天気 🌸\"}" } ], "exception": null } } } }, { "fixture_id": "llama/l3-empty-arguments", "family": "llama", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{}' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_time: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_time: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_time: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_time: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "llama3", "reasoning_parser": null, "tool_call_detector": "Llama32Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-3.3-70B-Instruct", "tokenizer": { "repo": "unsloth/Llama-3.3-70B-Instruct", "revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128006, 78191, 128007, 271 ] }, "stop": { "token": "<|end_of_text|>", "id": 128001, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null } } } }, { "fixture_id": "llama/l3-marker-in-arguments", "family": "llama", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '{\"name\": \"write_file\", \"parameters\": {\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with ; headers look like <|start_header_id|>ipython<|end_hea...; tool_calls: expected ['write_file'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['write_file'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with <|python_tag|>; headers look like <|start_header_id|>ipython<|end_header_id|>.\"}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with <|python_tag|>; headers look like <|start_header_id|>ipython<|end_header_id|>.\"}', got '{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with\"}'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with <|python_tag|>; headers look like <|start_header_id|>ipython<|end_header_id|>.\"}', got '{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start\"}'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with <|python_tag|>; headers look like <|start_header_id|>ipython<|end_header_id|>.\"}', got '{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with\"}'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with <|python_tag|>; headers look like <|start_header_id|>ipython<|end_header_id|>.\"}', got '{\"path\": \"docs/llama.md\", \"content\": \"\"}'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with <|python_tag|>; headers look like <|start_header_id|>ipython<|end_header_id|>.\"}', got '{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with\"}'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with <|python_tag|>; headers look like <|start_header_id|>ipython<|end_header_id|>.\"}', got '{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with\"}'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '{\"name\": \"write_file\", \"parameters\": {\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with ; headers look like <|start_header_id|>ipython<|end_hea..., got None" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '{\"name\": \"write_file\", \"parameters\": {\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with ; headers look like <|start_header_id|>ipython<|end_hea..., got None; tool_calls: nonstream [], got ['write_file']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '{\"name\": \"write_file\", \"parameters\": {\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with ; headers look like <|start_header_id|>ipython<|end_hea..., got None; tool_calls: nonstream [], got ['write_file']" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '{\"name\": \"write_file\", \"parameters\": {\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with ; headers look like <|start_header_id|>ipython<|end_hea..., got None; tool_calls: nonstream [], got ['write_file']" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '{\"name\": \"write_file\", \"parameters\": {\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with ; headers look like <|start_header_id|>ipython<|end_hea..., got None; tool_calls: nonstream [], got ['write_file']" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '{\"name\": \"write_file\", \"parameters\": {\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with ; headers look like <|start_header_id|>ipython<|end_hea..., got None; tool_calls: nonstream [], got ['write_file']" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '{\"name\": \"write_file\", \"parameters\": {\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with ; headers look like <|start_header_id|>ipython<|end_hea..., got None; tool_calls: nonstream [], got ['write_file']" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '{\"name\": \"write_file\", \"parameters\": {\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with ; headers look like <|start_header_id|>ipython<|end_hea..., got None; tool_calls: nonstream [], got ['write_file']" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: special (fail): tool_calls: one [], got ['write_file'] | token (fail): tool_calls: one [], got ['write_file'] | rand:1:8 (fail): tool_calls: one [], got ['write_file'] | rand:2:8 (fail): tool_calls: one [], got ['write_file'] | rand:3:8 (fail): tool_calls: one [], got ['write_file'] | rand:4:8 (fail): tool_calls: one [], got ['write_file'] | rand:5:8 (fail): tool_calls: one [], got ['write_file']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '<|start_header_id|>'; content contains '<|end_header_id|>'" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "llama3", "reasoning_parser": null, "tool_call_detector": "Llama32Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-3.3-70B-Instruct", "tokenizer": { "repo": "unsloth/Llama-3.3-70B-Instruct", "revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128006, 78191, 128007, 271 ] }, "stop": { "token": "<|end_of_text|>", "id": 128001, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "{\"name\": \"write_file\", \"parameters\": {\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with ; headers look like <|start_header_id|>ipython<|end_header_id|>.\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"docs/llama.md\", \"content\": \"\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"docs/llama.md\", \"content\": \"Tool calls may start with\"}" } ], "exception": null } } } }, { "fixture_id": "llama/l3-string-escapes", "family": "llama", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"C:\\\\\\\\Users\\\\\\\\ana\\\\\\\\notes.txt\", \"content\": \"line 1\\\\n\\\\tline 2 \\\\\"quoted\\\\\" {braces} \\\\\\\\ end\"}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"C:\\\\\\\\Users\\\\\\\\ana\\\\\\\\notes.txt\", \"content\": \"line 1\\\\n\\\\tline 2 \\\\\"quoted\\\\\" {braces} \\\\\\\\ end\"}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"path\": \"C:\\\\\\\\Users\\\\\\\\ana\\\\\\\\notes.txt\", \"content\": \"line 1\\\\n\\\\tline 2 \\\\\"quoted\\\\\" {braces} \\\\\\\\ end\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"path\": \"C:\\\\\\\\Users\\\\\\\\ana\\\\\\\\notes.txt\", \"content\": \"line 1\\\\n\\\\tline 2 \\\\\"quoted\\\\\" {braces} \\\\\\\\ end\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{\"path\": \"C:\\\\\\\\Users\\\\\\\\ana\\\\\\\\notes.txt\", \"content\": \"line 1\\\\n\\\\tline 2 \\\\\"quoted\\\\\" {braces} \\\\\\\\ end\"}' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"path\": \"C:\\\\\\\\Users\\\\\\\\ana\\\\\\\\notes.txt\", \"content\": \"line 1\\\\n\\\\tline 2 \\\\\"quoted\\\\\" {braces} \\\\\\\\ end\"}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{\"path\": \"C:\\\\\\\\Users\\\\\\\\ana\\\\\\\\notes.txt\", \"content\": \"line 1\\\\n\\\\tline 2 \\\\\"quoted\\\\\" {braces} \\\\\\\\ end\"}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"path\": \"C:\\\\\\\\Users\\\\\\\\ana\\\\\\\\notes.txt\", \"content\": \"line 1\\\\n\\\\tline 2 \\\\\"quoted\\\\\" {braces} \\\\\\\\ end\"}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"path\": \"C:\\\\\\\\Users\\\\\\\\ana\\\\\\\\notes.txt\", \"content\": \"line 1\\\\n\\\\tline 2 \\\\\"quoted\\\\\" {braces} \\\\\\\\ end\"}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"path\": \"C:\\\\\\\\Users\\\\\\\\ana\\\\\\\\notes.txt\", \"content\": \"line 1\\\\n\\\\tline 2 \\\\\"quoted\\\\\" {braces} \\\\\\\\ end\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "llama3", "reasoning_parser": null, "tool_call_detector": "Llama32Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-3.3-70B-Instruct", "tokenizer": { "repo": "unsloth/Llama-3.3-70B-Instruct", "revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128006, 78191, 128007, 271 ] }, "stop": { "token": "<|end_of_text|>", "id": 128001, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"C:\\\\Users\\\\ana\\\\notes.txt\", \"content\": \"line 1\\n\\tline 2 \\\"quoted\\\" {braces} \\\\ end\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"C:\\\\Users\\\\ana\\\\notes.txt\", \"content\": \"line 1\\n\\tline 2 \\\"quoted\\\" {braces} \\\\ end\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"C:\\\\Users\\\\ana\\\\notes.txt\", \"content\": \"line 1\\n\\tline 2 \\\"quoted\\\" {braces} \\\\ end\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"C:\\\\Users\\\\ana\\\\notes.txt\", \"content\": \"line 1\\n\\tline 2 \\\"quoted\\\" {braces} \\\\ end\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"C:\\\\Users\\\\ana\\\\notes.txt\", \"content\": \"line 1\\n\\tline 2 \\\"quoted\\\" {braces} \\\\ end\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"C:\\\\Users\\\\ana\\\\notes.txt\", \"content\": \"line 1\\n\\tline 2 \\\"quoted\\\" {braces} \\\\ end\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"C:\\\\Users\\\\ana\\\\notes.txt\", \"content\": \"line 1\\n\\tline 2 \\\"quoted\\\" {braces} \\\\ end\"}" } ], "exception": null } } } }, { "fixture_id": "llama/l3-typed-values", "family": "llama", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15, \"all_day\": false, \"reminder_minutes\": null}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15, \"all_day\": false, \"reminder_minutes\": null}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15, \"all_day\": false, \"reminder_minutes\": null}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15, \"all_day\": false, \"reminder_minutes\": null}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15, \"all_day\": false, \"reminder_minutes\": null}' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15, \"all_day\": false, \"reminder_minutes\": null}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15, \"all_day\": false, \"reminder_minutes\": null}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15, \"all_day\": false, \"reminder_minutes\": null}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15, \"all_day\": false, \"reminder_minutes\": null}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15, \"all_day\": false, \"reminder_minutes\": null}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] create_event: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] create_event: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] create_event: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] create_event: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "llama3", "reasoning_parser": null, "tool_call_detector": "Llama32Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-3.3-70B-Instruct", "tokenizer": { "repo": "unsloth/Llama-3.3-70B-Instruct", "revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128006, 78191, 128007, 271 ] }, "stop": { "token": "<|end_of_text|>", "id": 128001, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15, \"all_day\": false, \"reminder_minutes\": null}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15, \"all_day\": false, \"reminder_minutes\": null}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15, \"all_day\": false, \"reminder_minutes\": null}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15, \"all_day\": false, \"reminder_minutes\": null}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15, \"all_day\": false, \"reminder_minutes\": null}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15, \"all_day\": false, \"reminder_minutes\": null}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15, \"all_day\": false, \"reminder_minutes\": null}" } ], "exception": null } } } }, { "fixture_id": "llama/l3-long-arguments", "family": "llama", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"fox.txt\", \"content\": \"000: The quick brown fox jumps over the lazy dog.\\\\n001: The quick brown fox jumps over the lazy dog.\\\\n002: The quick brown..., got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"fox.txt\", \"content\": \"000: The quick brown fox jumps over the lazy dog.\\\\n001: The quick brown fox jumps over the lazy dog.\\\\n002: The quick brown..., got ''" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"path\": \"fox.txt\", \"content\": \"000: The quick brown fox jumps over the lazy dog.\\\\n001: The quick brown fox jumps over the lazy dog.\\\\n002: The quick brown..., got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"path\": \"fox.txt\", \"content\": \"000: The quick brown fox jumps over the lazy dog.\\\\n001: The quick brown fox jumps over the lazy dog.\\\\n002: The quick brown..., got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{\"path\": \"fox.txt\", \"content\": \"000: The quick brown fox jumps over the lazy dog.\\\\n001: The quick brown fox jumps over the lazy dog.\\\\n002: The quick brown... | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"path\": \"fox.txt\", \"content\": \"000: The quick brown fox jumps over the lazy dog.\\\\n001: The quick brown fox jumps over the lazy dog.\\\\n002: The quick brown... | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{\"path\": \"fox.txt\", \"content\": \"000: The quick brown fox jumps over the lazy dog.\\\\n001: The quick brown fox jumps over the lazy dog.\\\\n002: The quick brown... | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"path\": \"fox.txt\", \"content\": \"000: The quick brown fox jumps over the lazy dog.\\\\n001: The quick brown fox jumps over the lazy dog.\\\\n002: The quick brown... | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"path\": \"fox.txt\", \"content\": \"000: The quick brown fox jumps over the lazy dog.\\\\n001: The quick brown fox jumps over the lazy dog.\\\\n002: The quick brown... | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"path\": \"fox.txt\", \"content\": \"000: The quick brown fox jumps over the lazy dog.\\\\n001: The quick brown fox jumps over the lazy dog.\\\\n002: The quick brown..." }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "llama3", "reasoning_parser": null, "tool_call_detector": "Llama32Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-3.3-70B-Instruct", "tokenizer": { "repo": "unsloth/Llama-3.3-70B-Instruct", "revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128006, 78191, 128007, 271 ] }, "stop": { "token": "<|end_of_text|>", "id": 128001, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"fox.txt\", \"content\": \"000: The quick brown fox jumps over the lazy dog.\\n001: The quick brown fox jumps over the lazy dog.\\n002: The quick brown fox jumps over the lazy dog.\\n003: The quick brown fox jumps over the lazy dog.\\n004: The quick brown fox jumps over the lazy dog.\\n005: The quick brown fox jumps over the lazy dog.\\n006: The quick brown fox jumps over the lazy dog.\\n007: The quick brown fox jumps over the lazy dog.\\n008: The quick brown fox jumps over the lazy dog.\\n009: The quick brown fox jumps over the lazy dog.\\n010: The quick brown fox jumps over the lazy dog.\\n011: The quick brown fox jumps over the lazy dog.\\n012: The quick brown fox jumps over the lazy dog.\\n013: The quick brown fox jumps over the lazy dog.\\n014: The quick brown fox jumps over the lazy dog.\\n015: The quick brown fox jumps over the lazy dog.\\n016: The quick brown fox jumps over the lazy dog.\\n017: The quick brown fox jumps over the lazy dog.\\n018: The quick brown fox jumps over the lazy dog.\\n019: The quick brown fox jumps over the lazy dog.\\n020: The quick brown fox jumps over the lazy dog.\\n021: The quick brown fox jumps over the lazy dog.\\n022: The quick brown fox jumps over the lazy dog.\\n023: The quick brown fox jumps over the lazy dog.\\n024: The quick brown fox jumps over the lazy dog.\\n025: The quick brown fox jumps over the lazy dog.\\n026: The quick brown fox jumps over the lazy dog.\\n027: The quick brown fox jumps over the lazy dog.\\n028: The quick brown fox jumps over the lazy dog.\\n029: The quick brown fox jumps over the lazy dog.\\n030: The quick brown fox jumps over the lazy dog.\\n031: The quick brown fox jumps over the lazy dog.\\n032: The quick brown fox jumps over the lazy dog.\\n033: The quick brown fox jumps over the lazy dog.\\n034: The quick brown fox jumps over the lazy dog.\\n035: The quick brown fox jumps over the lazy dog.\\n036: The quick brown fox jumps over the lazy dog.\\n037: The quick brown fox jumps over the lazy dog.\\n038: The quick brown fox jumps over the lazy dog.\\n039: The quick brown fox jumps over the lazy dog.\\n040: The quick brown fox jumps over the lazy dog.\\n041: The quick brown fox jumps over the lazy dog.\\n042: The quick brown fox jumps over the lazy dog.\\n043: The quick brown fox jumps over the lazy dog.\\n044: The quick brown fox jumps over the lazy dog.\\n045: The quick brown fox jumps over the lazy dog.\\n046: The quick brown fox jumps over the lazy dog.\\n047: The quick brown fox jumps over the lazy dog.\\n048: The quick brown fox jumps over the lazy dog.\\n049: The quick brown fox jumps over the lazy dog.\\n050: The quick brown fox jumps over the lazy dog.\\n051: The quick brown fox jumps over the lazy dog.\\n052: The quick brown fox jumps over the lazy dog.\\n053: The quick brown fox jumps over the lazy dog.\\n054: The quick brown fox jumps over the lazy dog.\\n055: The quick brown fox jumps over the lazy dog.\\n056: The quick brown fox jumps over the lazy dog.\\n057: The quick brown fox jumps over the lazy dog.\\n058: The quick brown fox jumps over the lazy dog.\\n059: The quick brown fox jumps over the lazy dog.\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"fox.txt\", \"content\": \"000: The quick brown fox jumps over the lazy dog.\\n001: The quick brown fox jumps over the lazy dog.\\n002: The quick brown fox jumps over the lazy dog.\\n003: The quick brown fox jumps over the lazy dog.\\n004: The quick brown fox jumps over the lazy dog.\\n005: The quick brown fox jumps over the lazy dog.\\n006: The quick brown fox jumps over the lazy dog.\\n007: The quick brown fox jumps over the lazy dog.\\n008: The quick brown fox jumps over the lazy dog.\\n009: The quick brown fox jumps over the lazy dog.\\n010: The quick brown fox jumps over the lazy dog.\\n011: The quick brown fox jumps over the lazy dog.\\n012: The quick brown fox jumps over the lazy dog.\\n013: The quick brown fox jumps over the lazy dog.\\n014: The quick brown fox jumps over the lazy dog.\\n015: The quick brown fox jumps over the lazy dog.\\n016: The quick brown fox jumps over the lazy dog.\\n017: The quick brown fox jumps over the lazy dog.\\n018: The quick brown fox jumps over the lazy dog.\\n019: The quick brown fox jumps over the lazy dog.\\n020: The quick brown fox jumps over the lazy dog.\\n021: The quick brown fox jumps over the lazy dog.\\n022: The quick brown fox jumps over the lazy dog.\\n023: The quick brown fox jumps over the lazy dog.\\n024: The quick brown fox jumps over the lazy dog.\\n025: The quick brown fox jumps over the lazy dog.\\n026: The quick brown fox jumps over the lazy dog.\\n027: The quick brown fox jumps over the lazy dog.\\n028: The quick brown fox jumps over the lazy dog.\\n029: The quick brown fox jumps over the lazy dog.\\n030: The quick brown fox jumps over the lazy dog.\\n031: The quick brown fox jumps over the lazy dog.\\n032: The quick brown fox jumps over the lazy dog.\\n033: The quick brown fox jumps over the lazy dog.\\n034: The quick brown fox jumps over the lazy dog.\\n035: The quick brown fox jumps over the lazy dog.\\n036: The quick brown fox jumps over the lazy dog.\\n037: The quick brown fox jumps over the lazy dog.\\n038: The quick brown fox jumps over the lazy dog.\\n039: The quick brown fox jumps over the lazy dog.\\n040: The quick brown fox jumps over the lazy dog.\\n041: The quick brown fox jumps over the lazy dog.\\n042: The quick brown fox jumps over the lazy dog.\\n043: The quick brown fox jumps over the lazy dog.\\n044: The quick brown fox jumps over the lazy dog.\\n045: The quick brown fox jumps over the lazy dog.\\n046: The quick brown fox jumps over the lazy dog.\\n047: The quick brown fox jumps over the lazy dog.\\n048: The quick brown fox jumps over the lazy dog.\\n049: The quick brown fox jumps over the lazy dog.\\n050: The quick brown fox jumps over the lazy dog.\\n051: The quick brown fox jumps over the lazy dog.\\n052: The quick brown fox jumps over the lazy dog.\\n053: The quick brown fox jumps over the lazy dog.\\n054: The quick brown fox jumps over the lazy dog.\\n055: The quick brown fox jumps over the lazy dog.\\n056: The quick brown fox jumps over the lazy dog.\\n057: The quick brown fox jumps over the lazy dog.\\n058: The quick brown fox jumps over the lazy dog.\\n059: The quick brown fox jumps over the lazy dog.\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"fox.txt\", \"content\": \"000: The quick brown fox jumps over the lazy dog.\\n001: The quick brown fox jumps over the lazy dog.\\n002: The quick brown fox jumps over the lazy dog.\\n003: The quick brown fox jumps over the lazy dog.\\n004: The quick brown fox jumps over the lazy dog.\\n005: The quick brown fox jumps over the lazy dog.\\n006: The quick brown fox jumps over the lazy dog.\\n007: The quick brown fox jumps over the lazy dog.\\n008: The quick brown fox jumps over the lazy dog.\\n009: The quick brown fox jumps over the lazy dog.\\n010: The quick brown fox jumps over the lazy dog.\\n011: The quick brown fox jumps over the lazy dog.\\n012: The quick brown fox jumps over the lazy dog.\\n013: The quick brown fox jumps over the lazy dog.\\n014: The quick brown fox jumps over the lazy dog.\\n015: The quick brown fox jumps over the lazy dog.\\n016: The quick brown fox jumps over the lazy dog.\\n017: The quick brown fox jumps over the lazy dog.\\n018: The quick brown fox jumps over the lazy dog.\\n019: The quick brown fox jumps over the lazy dog.\\n020: The quick brown fox jumps over the lazy dog.\\n021: The quick brown fox jumps over the lazy dog.\\n022: The quick brown fox jumps over the lazy dog.\\n023: The quick brown fox jumps over the lazy dog.\\n024: The quick brown fox jumps over the lazy dog.\\n025: The quick brown fox jumps over the lazy dog.\\n026: The quick brown fox jumps over the lazy dog.\\n027: The quick brown fox jumps over the lazy dog.\\n028: The quick brown fox jumps over the lazy dog.\\n029: The quick brown fox jumps over the lazy dog.\\n030: The quick brown fox jumps over the lazy dog.\\n031: The quick brown fox jumps over the lazy dog.\\n032: The quick brown fox jumps over the lazy dog.\\n033: The quick brown fox jumps over the lazy dog.\\n034: The quick brown fox jumps over the lazy dog.\\n035: The quick brown fox jumps over the lazy dog.\\n036: The quick brown fox jumps over the lazy dog.\\n037: The quick brown fox jumps over the lazy dog.\\n038: The quick brown fox jumps over the lazy dog.\\n039: The quick brown fox jumps over the lazy dog.\\n040: The quick brown fox jumps over the lazy dog.\\n041: The quick brown fox jumps over the lazy dog.\\n042: The quick brown fox jumps over the lazy dog.\\n043: The quick brown fox jumps over the lazy dog.\\n044: The quick brown fox jumps over the lazy dog.\\n045: The quick brown fox jumps over the lazy dog.\\n046: The quick brown fox jumps over the lazy dog.\\n047: The quick brown fox jumps over the lazy dog.\\n048: The quick brown fox jumps over the lazy dog.\\n049: The quick brown fox jumps over the lazy dog.\\n050: The quick brown fox jumps over the lazy dog.\\n051: The quick brown fox jumps over the lazy dog.\\n052: The quick brown fox jumps over the lazy dog.\\n053: The quick brown fox jumps over the lazy dog.\\n054: The quick brown fox jumps over the lazy dog.\\n055: The quick brown fox jumps over the lazy dog.\\n056: The quick brown fox jumps over the lazy dog.\\n057: The quick brown fox jumps over the lazy dog.\\n058: The quick brown fox jumps over the lazy dog.\\n059: The quick brown fox jumps over the lazy dog.\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"fox.txt\", \"content\": \"000: The quick brown fox jumps over the lazy dog.\\n001: The quick brown fox jumps over the lazy dog.\\n002: The quick brown fox jumps over the lazy dog.\\n003: The quick brown fox jumps over the lazy dog.\\n004: The quick brown fox jumps over the lazy dog.\\n005: The quick brown fox jumps over the lazy dog.\\n006: The quick brown fox jumps over the lazy dog.\\n007: The quick brown fox jumps over the lazy dog.\\n008: The quick brown fox jumps over the lazy dog.\\n009: The quick brown fox jumps over the lazy dog.\\n010: The quick brown fox jumps over the lazy dog.\\n011: The quick brown fox jumps over the lazy dog.\\n012: The quick brown fox jumps over the lazy dog.\\n013: The quick brown fox jumps over the lazy dog.\\n014: The quick brown fox jumps over the lazy dog.\\n015: The quick brown fox jumps over the lazy dog.\\n016: The quick brown fox jumps over the lazy dog.\\n017: The quick brown fox jumps over the lazy dog.\\n018: The quick brown fox jumps over the lazy dog.\\n019: The quick brown fox jumps over the lazy dog.\\n020: The quick brown fox jumps over the lazy dog.\\n021: The quick brown fox jumps over the lazy dog.\\n022: The quick brown fox jumps over the lazy dog.\\n023: The quick brown fox jumps over the lazy dog.\\n024: The quick brown fox jumps over the lazy dog.\\n025: The quick brown fox jumps over the lazy dog.\\n026: The quick brown fox jumps over the lazy dog.\\n027: The quick brown fox jumps over the lazy dog.\\n028: The quick brown fox jumps over the lazy dog.\\n029: The quick brown fox jumps over the lazy dog.\\n030: The quick brown fox jumps over the lazy dog.\\n031: The quick brown fox jumps over the lazy dog.\\n032: The quick brown fox jumps over the lazy dog.\\n033: The quick brown fox jumps over the lazy dog.\\n034: The quick brown fox jumps over the lazy dog.\\n035: The quick brown fox jumps over the lazy dog.\\n036: The quick brown fox jumps over the lazy dog.\\n037: The quick brown fox jumps over the lazy dog.\\n038: The quick brown fox jumps over the lazy dog.\\n039: The quick brown fox jumps over the lazy dog.\\n040: The quick brown fox jumps over the lazy dog.\\n041: The quick brown fox jumps over the lazy dog.\\n042: The quick brown fox jumps over the lazy dog.\\n043: The quick brown fox jumps over the lazy dog.\\n044: The quick brown fox jumps over the lazy dog.\\n045: The quick brown fox jumps over the lazy dog.\\n046: The quick brown fox jumps over the lazy dog.\\n047: The quick brown fox jumps over the lazy dog.\\n048: The quick brown fox jumps over the lazy dog.\\n049: The quick brown fox jumps over the lazy dog.\\n050: The quick brown fox jumps over the lazy dog.\\n051: The quick brown fox jumps over the lazy dog.\\n052: The quick brown fox jumps over the lazy dog.\\n053: The quick brown fox jumps over the lazy dog.\\n054: The quick brown fox jumps over the lazy dog.\\n055: The quick brown fox jumps over the lazy dog.\\n056: The quick brown fox jumps over the lazy dog.\\n057: The quick brown fox jumps over the lazy dog.\\n058: The quick brown fox jumps over the lazy dog.\\n059: The quick brown fox jumps over the lazy dog.\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"fox.txt\", \"content\": \"000: The quick brown fox jumps over the lazy dog.\\n001: The quick brown fox jumps over the lazy dog.\\n002: The quick brown fox jumps over the lazy dog.\\n003: The quick brown fox jumps over the lazy dog.\\n004: The quick brown fox jumps over the lazy dog.\\n005: The quick brown fox jumps over the lazy dog.\\n006: The quick brown fox jumps over the lazy dog.\\n007: The quick brown fox jumps over the lazy dog.\\n008: The quick brown fox jumps over the lazy dog.\\n009: The quick brown fox jumps over the lazy dog.\\n010: The quick brown fox jumps over the lazy dog.\\n011: The quick brown fox jumps over the lazy dog.\\n012: The quick brown fox jumps over the lazy dog.\\n013: The quick brown fox jumps over the lazy dog.\\n014: The quick brown fox jumps over the lazy dog.\\n015: The quick brown fox jumps over the lazy dog.\\n016: The quick brown fox jumps over the lazy dog.\\n017: The quick brown fox jumps over the lazy dog.\\n018: The quick brown fox jumps over the lazy dog.\\n019: The quick brown fox jumps over the lazy dog.\\n020: The quick brown fox jumps over the lazy dog.\\n021: The quick brown fox jumps over the lazy dog.\\n022: The quick brown fox jumps over the lazy dog.\\n023: The quick brown fox jumps over the lazy dog.\\n024: The quick brown fox jumps over the lazy dog.\\n025: The quick brown fox jumps over the lazy dog.\\n026: The quick brown fox jumps over the lazy dog.\\n027: The quick brown fox jumps over the lazy dog.\\n028: The quick brown fox jumps over the lazy dog.\\n029: The quick brown fox jumps over the lazy dog.\\n030: The quick brown fox jumps over the lazy dog.\\n031: The quick brown fox jumps over the lazy dog.\\n032: The quick brown fox jumps over the lazy dog.\\n033: The quick brown fox jumps over the lazy dog.\\n034: The quick brown fox jumps over the lazy dog.\\n035: The quick brown fox jumps over the lazy dog.\\n036: The quick brown fox jumps over the lazy dog.\\n037: The quick brown fox jumps over the lazy dog.\\n038: The quick brown fox jumps over the lazy dog.\\n039: The quick brown fox jumps over the lazy dog.\\n040: The quick brown fox jumps over the lazy dog.\\n041: The quick brown fox jumps over the lazy dog.\\n042: The quick brown fox jumps over the lazy dog.\\n043: The quick brown fox jumps over the lazy dog.\\n044: The quick brown fox jumps over the lazy dog.\\n045: The quick brown fox jumps over the lazy dog.\\n046: The quick brown fox jumps over the lazy dog.\\n047: The quick brown fox jumps over the lazy dog.\\n048: The quick brown fox jumps over the lazy dog.\\n049: The quick brown fox jumps over the lazy dog.\\n050: The quick brown fox jumps over the lazy dog.\\n051: The quick brown fox jumps over the lazy dog.\\n052: The quick brown fox jumps over the lazy dog.\\n053: The quick brown fox jumps over the lazy dog.\\n054: The quick brown fox jumps over the lazy dog.\\n055: The quick brown fox jumps over the lazy dog.\\n056: The quick brown fox jumps over the lazy dog.\\n057: The quick brown fox jumps over the lazy dog.\\n058: The quick brown fox jumps over the lazy dog.\\n059: The quick brown fox jumps over the lazy dog.\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"fox.txt\", \"content\": \"000: The quick brown fox jumps over the lazy dog.\\n001: The quick brown fox jumps over the lazy dog.\\n002: The quick brown fox jumps over the lazy dog.\\n003: The quick brown fox jumps over the lazy dog.\\n004: The quick brown fox jumps over the lazy dog.\\n005: The quick brown fox jumps over the lazy dog.\\n006: The quick brown fox jumps over the lazy dog.\\n007: The quick brown fox jumps over the lazy dog.\\n008: The quick brown fox jumps over the lazy dog.\\n009: The quick brown fox jumps over the lazy dog.\\n010: The quick brown fox jumps over the lazy dog.\\n011: The quick brown fox jumps over the lazy dog.\\n012: The quick brown fox jumps over the lazy dog.\\n013: The quick brown fox jumps over the lazy dog.\\n014: The quick brown fox jumps over the lazy dog.\\n015: The quick brown fox jumps over the lazy dog.\\n016: The quick brown fox jumps over the lazy dog.\\n017: The quick brown fox jumps over the lazy dog.\\n018: The quick brown fox jumps over the lazy dog.\\n019: The quick brown fox jumps over the lazy dog.\\n020: The quick brown fox jumps over the lazy dog.\\n021: The quick brown fox jumps over the lazy dog.\\n022: The quick brown fox jumps over the lazy dog.\\n023: The quick brown fox jumps over the lazy dog.\\n024: The quick brown fox jumps over the lazy dog.\\n025: The quick brown fox jumps over the lazy dog.\\n026: The quick brown fox jumps over the lazy dog.\\n027: The quick brown fox jumps over the lazy dog.\\n028: The quick brown fox jumps over the lazy dog.\\n029: The quick brown fox jumps over the lazy dog.\\n030: The quick brown fox jumps over the lazy dog.\\n031: The quick brown fox jumps over the lazy dog.\\n032: The quick brown fox jumps over the lazy dog.\\n033: The quick brown fox jumps over the lazy dog.\\n034: The quick brown fox jumps over the lazy dog.\\n035: The quick brown fox jumps over the lazy dog.\\n036: The quick brown fox jumps over the lazy dog.\\n037: The quick brown fox jumps over the lazy dog.\\n038: The quick brown fox jumps over the lazy dog.\\n039: The quick brown fox jumps over the lazy dog.\\n040: The quick brown fox jumps over the lazy dog.\\n041: The quick brown fox jumps over the lazy dog.\\n042: The quick brown fox jumps over the lazy dog.\\n043: The quick brown fox jumps over the lazy dog.\\n044: The quick brown fox jumps over the lazy dog.\\n045: The quick brown fox jumps over the lazy dog.\\n046: The quick brown fox jumps over the lazy dog.\\n047: The quick brown fox jumps over the lazy dog.\\n048: The quick brown fox jumps over the lazy dog.\\n049: The quick brown fox jumps over the lazy dog.\\n050: The quick brown fox jumps over the lazy dog.\\n051: The quick brown fox jumps over the lazy dog.\\n052: The quick brown fox jumps over the lazy dog.\\n053: The quick brown fox jumps over the lazy dog.\\n054: The quick brown fox jumps over the lazy dog.\\n055: The quick brown fox jumps over the lazy dog.\\n056: The quick brown fox jumps over the lazy dog.\\n057: The quick brown fox jumps over the lazy dog.\\n058: The quick brown fox jumps over the lazy dog.\\n059: The quick brown fox jumps over the lazy dog.\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"fox.txt\", \"content\": \"000: The quick brown fox jumps over the lazy dog.\\n001: The quick brown fox jumps over the lazy dog.\\n002: The quick brown fox jumps over the lazy dog.\\n003: The quick brown fox jumps over the lazy dog.\\n004: The quick brown fox jumps over the lazy dog.\\n005: The quick brown fox jumps over the lazy dog.\\n006: The quick brown fox jumps over the lazy dog.\\n007: The quick brown fox jumps over the lazy dog.\\n008: The quick brown fox jumps over the lazy dog.\\n009: The quick brown fox jumps over the lazy dog.\\n010: The quick brown fox jumps over the lazy dog.\\n011: The quick brown fox jumps over the lazy dog.\\n012: The quick brown fox jumps over the lazy dog.\\n013: The quick brown fox jumps over the lazy dog.\\n014: The quick brown fox jumps over the lazy dog.\\n015: The quick brown fox jumps over the lazy dog.\\n016: The quick brown fox jumps over the lazy dog.\\n017: The quick brown fox jumps over the lazy dog.\\n018: The quick brown fox jumps over the lazy dog.\\n019: The quick brown fox jumps over the lazy dog.\\n020: The quick brown fox jumps over the lazy dog.\\n021: The quick brown fox jumps over the lazy dog.\\n022: The quick brown fox jumps over the lazy dog.\\n023: The quick brown fox jumps over the lazy dog.\\n024: The quick brown fox jumps over the lazy dog.\\n025: The quick brown fox jumps over the lazy dog.\\n026: The quick brown fox jumps over the lazy dog.\\n027: The quick brown fox jumps over the lazy dog.\\n028: The quick brown fox jumps over the lazy dog.\\n029: The quick brown fox jumps over the lazy dog.\\n030: The quick brown fox jumps over the lazy dog.\\n031: The quick brown fox jumps over the lazy dog.\\n032: The quick brown fox jumps over the lazy dog.\\n033: The quick brown fox jumps over the lazy dog.\\n034: The quick brown fox jumps over the lazy dog.\\n035: The quick brown fox jumps over the lazy dog.\\n036: The quick brown fox jumps over the lazy dog.\\n037: The quick brown fox jumps over the lazy dog.\\n038: The quick brown fox jumps over the lazy dog.\\n039: The quick brown fox jumps over the lazy dog.\\n040: The quick brown fox jumps over the lazy dog.\\n041: The quick brown fox jumps over the lazy dog.\\n042: The quick brown fox jumps over the lazy dog.\\n043: The quick brown fox jumps over the lazy dog.\\n044: The quick brown fox jumps over the lazy dog.\\n045: The quick brown fox jumps over the lazy dog.\\n046: The quick brown fox jumps over the lazy dog.\\n047: The quick brown fox jumps over the lazy dog.\\n048: The quick brown fox jumps over the lazy dog.\\n049: The quick brown fox jumps over the lazy dog.\\n050: The quick brown fox jumps over the lazy dog.\\n051: The quick brown fox jumps over the lazy dog.\\n052: The quick brown fox jumps over the lazy dog.\\n053: The quick brown fox jumps over the lazy dog.\\n054: The quick brown fox jumps over the lazy dog.\\n055: The quick brown fox jumps over the lazy dog.\\n056: The quick brown fox jumps over the lazy dog.\\n057: The quick brown fox jumps over the lazy dog.\\n058: The quick brown fox jumps over the lazy dog.\\n059: The quick brown fox jumps over the lazy dog.\"}" } ], "exception": null } } } }, { "fixture_id": "llama/l3-no-call", "family": "llama", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "llama3", "reasoning_parser": null, "tool_call_detector": "Llama32Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-3.3-70B-Instruct", "tokenizer": { "repo": "unsloth/Llama-3.3-70B-Instruct", "revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128006, 78191, 128007, 271 ] }, "stop": { "token": "<|end_of_text|>", "id": 128001, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "llama/l3-json-answer-not-a-call", "family": "llama", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected '{\"capital\": \"Paris\", \"country\": \"France\"}', got None" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected '{\"capital\": \"Paris\", \"country\": \"France\"}', got None" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected '{\"capital\": \"Paris\", \"country\": \"France\"}', got None" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected '{\"capital\": \"Paris\", \"country\": \"France\"}', got None" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected '{\"capital\": \"Paris\", \"country\": \"France\"}', got None" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected '{\"capital\": \"Paris\", \"country\": \"France\"}', got None" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected '{\"capital\": \"Paris\", \"country\": \"France\"}', got None" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected '{\"capital\": \"Paris\", \"country\": \"France\"}', got None" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected '{\"capital\": \"Paris\", \"country\": \"France\"}', got None" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "llama3", "reasoning_parser": null, "tool_call_detector": "Llama32Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-3.3-70B-Instruct", "tokenizer": { "repo": "unsloth/Llama-3.3-70B-Instruct", "revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128006, 78191, 128007, 271 ] }, "stop": { "token": "<|end_of_text|>", "id": 128001, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null } } } }, { "fixture_id": "llama/l3-multi-turn-second-call", "family": "llama", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"}' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] convert_units: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] convert_units: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] convert_units: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] convert_units: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "llama3", "reasoning_parser": null, "tool_call_detector": "Llama32Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-3.3-70B-Instruct", "tokenizer": { "repo": "unsloth/Llama-3.3-70B-Instruct", "revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128006, 78191, 128007, 271 ] }, "stop": { "token": "<|end_of_text|>", "id": 128001, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "convert_units", "arguments_raw": "{\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "convert_units", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "convert_units", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "convert_units", "arguments_raw": "{\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "convert_units", "arguments_raw": "{\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "convert_units", "arguments_raw": "{\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "convert_units", "arguments_raw": "{\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "convert_units", "arguments_raw": "{\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "convert_units", "arguments_raw": "{\"value\": 21.5, \"from_unit\": \"celsius\", \"to_unit\": \"fahrenheit\"}" } ], "exception": null } } } }, { "fixture_id": "llama/l3-meta-python-tag-type-function", "family": "llama", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"n\": \"10\", \"genre\": \"all\"}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"n\": \"10\", \"genre\": \"all\"}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"n\": \"10\", \"genre\": \"all\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"n\": \"10\", \"genre\": \"all\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{\"n\": \"10\", \"genre\": \"all\"}' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"n\": \"10\", \"genre\": \"all\"}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{\"n\": \"10\", \"genre\": \"all\"}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"n\": \"10\", \"genre\": \"all\"}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"n\": \"10\", \"genre\": \"all\"}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"n\": \"10\", \"genre\": \"all\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] trending_songs: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] trending_songs: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] trending_songs: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] trending_songs: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "llama3", "reasoning_parser": null, "tool_call_detector": "Llama32Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-3.3-70B-Instruct", "tokenizer": { "repo": "unsloth/Llama-3.3-70B-Instruct", "revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128006, 78191, 128007, 271 ] }, "stop": { "token": "<|end_of_text|>", "id": 128001, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "trending_songs", "arguments_raw": "{\"n\": \"10\", \"genre\": \"all\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "trending_songs", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "trending_songs", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "trending_songs", "arguments_raw": "{\"n\": \"10\", \"genre\": \"all\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "trending_songs", "arguments_raw": "{\"n\": \"10\", \"genre\": \"all\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "trending_songs", "arguments_raw": "{\"n\": \"10\", \"genre\": \"all\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "trending_songs", "arguments_raw": "{\"n\": \"10\", \"genre\": \"all\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "trending_songs", "arguments_raw": "{\"n\": \"10\", \"genre\": \"all\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "trending_songs", "arguments_raw": "{\"n\": \"10\", \"genre\": \"all\"}" } ], "exception": null } } } }, { "fixture_id": "llama/l4-meta-pythonic-parallel", "family": "llama", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "pythonic", "reasoning_parser": null, "tool_call_detector": "PythonicDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "tokenizer": { "repo": "unsloth/Llama-4-Scout-17B-16E-Instruct", "revision": "afd8e498c87bda51c7ea8ec68ea2f7c066e6340b", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "93c6a232af67e2b3bc9218179ec3fea5176ec88b2a4a4fb66e51feb2ef63c0bd", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200005, 140680, 200006, 368 ] }, "stop": { "token": "<|end_of_text|>", "id": 200001, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "llama/l4-meta-pythonic-int-and-string", "family": "llama", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "pythonic", "reasoning_parser": null, "tool_call_detector": "PythonicDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "tokenizer": { "repo": "unsloth/Llama-4-Scout-17B-16E-Instruct", "revision": "afd8e498c87bda51c7ea8ec68ea2f7c066e6340b", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "93c6a232af67e2b3bc9218179ec3fea5176ec88b2a4a4fb66e51feb2ef63c0bd", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200005, 140680, 200006, 368 ] }, "stop": { "token": "<|end_of_text|>", "id": 200001, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "llama/l4-truncated-missing-close-bracket", "family": "llama", "status": "fail", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '[get_weather(city=\"San Francisco\"), get_weather(city=\"Seattle', got None" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '[get_weather(city=\"San Francisco\"), get_weather(city=\"Seattle', got None" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '[get_weather(city=\"San Francisco\"), get_weather(city=\"Seattle', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '[get_weather(city=\"San Francisco\"), get_weather(city=\"Seattle', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '[get_weather(city=\"San Francisco\"), get_weather(city=\"Seattle', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '[get_weather(city=\"San Francisco\"), get_weather(city=\"Seattle', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '[get_weather(city=\"San Francisco\"), get_weather(city=\"Seattle', got None" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '[get_weather(city=\"San Francisco\"), get_weather(city=\"Seattle', got None" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "pythonic", "reasoning_parser": null, "tool_call_detector": "PythonicDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "tokenizer": { "repo": "unsloth/Llama-4-Scout-17B-16E-Instruct", "revision": "afd8e498c87bda51c7ea8ec68ea2f7c066e6340b", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "93c6a232af67e2b3bc9218179ec3fea5176ec88b2a4a4fb66e51feb2ef63c0bd", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 200005, 140680, 200006, 368 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "[get_weather(city=\"San Francisco\"), get_weather(city=\"Seattle", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null } } } }, { "fixture_id": "llama/l3-truncated-inside-parameters", "family": "llama", "status": "fail", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "fail", "detail": "returned 1 tool call(s) ['create_event'] for max_tokens hit inside the parameters object; the JSON never closes." }, { "check": "expected_error", "strategy": "special", "status": "fail", "detail": "returned 1 tool call(s) ['create_event'] for max_tokens hit inside the parameters object; the JSON never closes." }, { "check": "expected_error", "strategy": "token", "status": "fail", "detail": "returned 1 tool call(s) ['create_event'] for max_tokens hit inside the parameters object; the JSON never closes." }, { "check": "expected_error", "strategy": "rand:1:8", "status": "fail", "detail": "returned 1 tool call(s) ['create_event'] for max_tokens hit inside the parameters object; the JSON never closes." }, { "check": "expected_error", "strategy": "rand:2:8", "status": "fail", "detail": "returned 1 tool call(s) ['create_event'] for max_tokens hit inside the parameters object; the JSON never closes." }, { "check": "expected_error", "strategy": "rand:3:8", "status": "fail", "detail": "returned 1 tool call(s) ['create_event'] for max_tokens hit inside the parameters object; the JSON never closes." }, { "check": "expected_error", "strategy": "rand:4:8", "status": "fail", "detail": "returned 1 tool call(s) ['create_event'] for max_tokens hit inside the parameters object; the JSON never closes." }, { "check": "expected_error", "strategy": "rand:5:8", "status": "fail", "detail": "returned 1 tool call(s) ['create_event'] for max_tokens hit inside the parameters object; the JSON never closes." }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '{\"name\": \"create_event\", \"parameters\": {\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15,', got None; tool_calls: nonstream [], got ['create_event']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '{\"name\": \"create_event\", \"parameters\": {\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15,', got None; tool_calls: nonstream [], got ['create_event']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '{\"name\": \"create_event\", \"parameters\": {\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15,', got None; tool_calls: nonstream [], got ['create_event']" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '{\"name\": \"create_event\", \"parameters\": {\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15,', got None; tool_calls: nonstream [], got ['create_event']" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '{\"name\": \"create_event\", \"parameters\": {\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15,', got None; tool_calls: nonstream [], got ['create_event']" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '{\"name\": \"create_event\", \"parameters\": {\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15,', got None; tool_calls: nonstream [], got ['create_event']" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '{\"name\": \"create_event\", \"parameters\": {\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15,', got None; tool_calls: nonstream [], got ['create_event']" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '{\"name\": \"create_event\", \"parameters\": {\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15,', got None; tool_calls: nonstream [], got ['create_event']" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15}' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] create_event: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] create_event: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] create_event: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] create_event: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "llama3", "reasoning_parser": null, "tool_call_detector": "Llama32Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "meta-llama/Llama-3.3-70B-Instruct", "tokenizer": { "repo": "unsloth/Llama-3.3-70B-Instruct", "revision": "99cd0d2c829e92a67c844f9144c2509632e5c87f", "class": "TokenizersBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": null, "hf_config_error": "LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.", "chat_template_sha256": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 128006, 78191, 128007, 271 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "{\"name\": \"create_event\", \"parameters\": {\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15,", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Standup\", \"attendees\": [\"ana@example.com\", \"bo@example.com\"], \"duration_minutes\": 15}" } ], "exception": null } } } }, { "fixture_id": "mistral/vllm-v3-no-space", "family": "mistral", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "tool_calls: expected ['add'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['add'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['add'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls: expected ['add'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: expected ['add'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: expected ['add'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: expected ['add'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: expected ['add'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: expected ['add'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Mistral-7B-Instruct-v0.3", "tokenizer": { "repo": "mistralai/Mistral-7B-Instruct-v0.3", "revision": "c170c708c41dac9275d15a8fff4eca08d52bab71", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "MistralForCausalLM" ], "model_type": "mistral" }, "hf_config_error": null, "chat_template_sha256": "e16746b40344d6c5b5265988e0328a0bf7277be86f1c335156eae07e29c82826", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 4 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null } } } }, { "fixture_id": "mistral/vllm-v3-arguments-before-name", "family": "mistral", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}', got ''" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}', got ''" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}', got ''" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}', got ''" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_current_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_current_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "fail", "detail": "[0] get_current_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "fail", "detail": "[0] get_current_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "fail", "detail": "[0] get_current_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "fail", "detail": "[0] get_current_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_current_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_current_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "fail", "detail": "[0] get_current_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "fail", "detail": "[0] get_current_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[0] get_current_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "fail", "detail": "[0] get_current_weather: arguments are not a JSON object; not validated" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Mistral-7B-Instruct-v0.3", "tokenizer": { "repo": "mistralai/Mistral-7B-Instruct-v0.3", "revision": "c170c708c41dac9275d15a8fff4eca08d52bab71", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "MistralForCausalLM" ], "model_type": "mistral" }, "hf_config_error": null, "chat_template_sha256": "e16746b40344d6c5b5265988e0328a0bf7277be86f1c335156eae07e29c82826", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 4 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "" } ], "exception": null } } } }, { "fixture_id": "mistral/vllm-v3-name-inside-arguments", "family": "mistral", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"name\": \"John Doe\"}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"name\": \"John Doe\"}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"name\": \"John Doe\"}', got ''" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"name\": \"John Doe\"}', got ''" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"name\": \"John Doe\"}', got ''" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"name\": \"John Doe\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"name\": \"John Doe\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"name\": \"John Doe\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"name\": \"John Doe\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"name\": \"John Doe\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"name\": \"John Doe\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"name\": \"John Doe\"}', got ''" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{\"name\": \"John Doe\"}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"name\": \"John Doe\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_age: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_age: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "fail", "detail": "[0] get_age: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "fail", "detail": "[0] get_age: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "fail", "detail": "[0] get_age: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "fail", "detail": "[0] get_age: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_age: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_age: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "fail", "detail": "[0] get_age: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "fail", "detail": "[0] get_age: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[0] get_age: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "fail", "detail": "[0] get_age: arguments are not a JSON object; not validated" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Mistral-7B-Instruct-v0.3", "tokenizer": { "repo": "mistralai/Mistral-7B-Instruct-v0.3", "revision": "c170c708c41dac9275d15a8fff4eca08d52bab71", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "MistralForCausalLM" ], "model_type": "mistral" }, "hf_config_error": null, "chat_template_sha256": "e16746b40344d6c5b5265988e0328a0bf7277be86f1c335156eae07e29c82826", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 4 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_age", "arguments_raw": "{\"name\": \"John Doe\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_age", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_age", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_age", "arguments_raw": "{\"name\": \"John Doe\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_age", "arguments_raw": "" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_age", "arguments_raw": "" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_age", "arguments_raw": "{\"name\": \"John Doe\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_age", "arguments_raw": "" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_age", "arguments_raw": "" } ], "exception": null } } } }, { "fixture_id": "mistral/vllm-v3-parallel", "family": "mistral", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['add', 'get_current_weather'], got ['add']" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['add', 'get_current_weather'], got ['add']" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected None, got ', {\"name\": \"get_current_weather\", \"arguments\":{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}}'; tool_calls: expected ['add', 'get_current_weather'], got ['add']" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected None, got ', {\"name\": \"get_current_weather\", \"arguments\":{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}}'; tool_calls: expected ['add', 'get_current_weather'], got ['add']" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected None, got ', {\"name\": \"get_current_weather\", \"arguments\":{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}}'; tool_calls: expected ['add', 'get_current_weather'], got ['add']" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected None, got ', {\"name\": \"get_current_weather\", \"arguments\":{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}}'; tool_calls: expected ['add', 'get_current_weather'], got ['add']" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected None, got ', {\"name\": \"get_current_weather\", \"arguments\":{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}}'; tool_calls: expected ['add', 'get_current_weather'], got ['add']" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected None, got ', {\"name\": \"get_current_weather\", \"arguments\":{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}}'; tool_calls: expected ['add', 'get_current_weather'], got ['add']" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream ['add', 'get_current_weather'], got ['add']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls: nonstream ['add', 'get_current_weather'], got ['add']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream None, got ', {\"name\": \"get_current_weather\", \"arguments\":{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}}'; tool_calls: nonstream ['add', 'get_current_weather'], got ['add']" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream None, got ', {\"name\": \"get_current_weather\", \"arguments\":{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}}'; tool_calls: nonstream ['add', 'get_current_weather'], got ['add']" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream None, got ', {\"name\": \"get_current_weather\", \"arguments\":{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}}'; tool_calls: nonstream ['add', 'get_current_weather'], got ['add']" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream None, got ', {\"name\": \"get_current_weather\", \"arguments\":{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}}'; tool_calls: nonstream ['add', 'get_current_weather'], got ['add']" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream None, got ', {\"name\": \"get_current_weather\", \"arguments\":{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}}'; tool_calls: nonstream ['add', 'get_current_weather'], got ['add']" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream None, got ', {\"name\": \"get_current_weather\", \"arguments\":{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}}'; tool_calls: nonstream ['add', 'get_current_weather'], got ['add']" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content: one None, got ', {\"name\": \"get_current_weather\", \"arguments\":{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}}'; tool_calls[0].arguments: one '', got '{\"a\": 3.5, \"b\": 4}' | rand:1:8 (fail): content: one None, got ', {\"name\": \"get_current_weather\", \"arguments\":{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}}'; tool_calls[0].arguments: one '', got '{\"a\": 3.5, \"b\": 4}' | rand:2:8 (fail): content: one None, got ', {\"name\": \"get_current_weather\", \"arguments\":{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}}'; tool_calls[0].arguments: one '', got '{\"a\": 3.5, \"b\": 4}' | rand:3:8 (fail): content: one None, got ', {\"name\": \"get_current_weather\", \"arguments\":{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}}'; tool_calls[0].arguments: one '', got '{\"a\": 3.5, \"b\": 4}' | rand:4:8 (fail): content: one None, got ', {\"name\": \"get_current_weather\", \"arguments\":{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}}'; tool_calls[0].arguments: one '', got '{\"a\": 3.5, \"b\": 4}' | rand:5:8 (fail): content: one None, got ', {\"name\": \"get_current_weather\", \"arguments\":{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}}'; tool_calls[0].arguments: one '', got '{\"a\": 3.5, \"b\": 4}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] add: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] add: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] add: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] add: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "fail", "detail": "expected 2 calls ['add', 'get_current_weather'], got 1 ['add']" }, { "check": "parallel_order", "strategy": "special", "status": "fail", "detail": "expected 2 calls ['add', 'get_current_weather'], got 1 ['add']" }, { "check": "parallel_order", "strategy": "token", "status": "fail", "detail": "expected 2 calls ['add', 'get_current_weather'], got 1 ['add']" }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "fail", "detail": "expected 2 calls ['add', 'get_current_weather'], got 1 ['add']" }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "fail", "detail": "expected 2 calls ['add', 'get_current_weather'], got 1 ['add']" }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "fail", "detail": "expected 2 calls ['add', 'get_current_weather'], got 1 ['add']" }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "fail", "detail": "expected 2 calls ['add', 'get_current_weather'], got 1 ['add']" }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "fail", "detail": "expected 2 calls ['add', 'get_current_weather'], got 1 ['add']" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Mistral-7B-Instruct-v0.3", "tokenizer": { "repo": "mistralai/Mistral-7B-Instruct-v0.3", "revision": "c170c708c41dac9275d15a8fff4eca08d52bab71", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "MistralForCausalLM" ], "model_type": "mistral" }, "hf_config_error": null, "chat_template_sha256": "e16746b40344d6c5b5265988e0328a0bf7277be86f1c335156eae07e29c82826", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 4 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "{\"a\": 3.5, \"b\": 4}" }, { "name": "get_current_weather", "arguments_raw": "{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "" } ], "exception": null }, "token": { "content": ", {\"name\": \"get_current_weather\", \"arguments\":{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}}", "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "{\"a\": 3.5, \"b\": 4}" } ], "exception": null }, "rand:1:8": { "content": ", {\"name\": \"get_current_weather\", \"arguments\":{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}}", "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "{\"a\": 3.5, \"b\": 4}" } ], "exception": null }, "rand:2:8": { "content": ", {\"name\": \"get_current_weather\", \"arguments\":{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}}", "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "{\"a\": 3.5, \"b\": 4}" } ], "exception": null }, "rand:3:8": { "content": ", {\"name\": \"get_current_weather\", \"arguments\":{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}}", "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "{\"a\": 3.5, \"b\": 4}" } ], "exception": null }, "rand:4:8": { "content": ", {\"name\": \"get_current_weather\", \"arguments\":{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}}", "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "{\"a\": 3.5, \"b\": 4}" } ], "exception": null }, "rand:5:8": { "content": ", {\"name\": \"get_current_weather\", \"arguments\":{\"city\": \"San Francisco\", \"state\": \"CA\", \"unit\": \"celsius\"}}", "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "{\"a\": 3.5, \"b\": 4}" } ], "exception": null } } } }, { "fixture_id": "mistral/vllm-v3-content-before-call", "family": "mistral", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content (whitespace only): expected 'Hello', got ' Hello'; tool_calls: expected ['add'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content (whitespace only): expected 'Hello', got ' Hello'; tool_calls[0].arguments: expected '{\"a\": 1, \"b\": 2}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected 'Hello', got ' Hello'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Hello', got ' Hello'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Hello', got ' Hello'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Hello', got ' Hello'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Hello', got ' Hello'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Hello', got ' Hello'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content (whitespace only): nonstream 'Hello', got ' Hello'; tool_calls: nonstream ['add'], got []" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content (whitespace only): nonstream 'Hello', got ' Hello'; tool_calls[0].arguments: nonstream '{\"a\": 1, \"b\": 2}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Hello', got ' Hello'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Hello', got ' Hello'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Hello', got ' Hello'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Hello', got ' Hello'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Hello', got ' Hello'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Hello', got ' Hello'" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: special (fail): tool_calls: one [], got ['add'] | token (fail): tool_calls: one [], got ['add'] | rand:1:8 (fail): tool_calls: one [], got ['add'] | rand:2:8 (fail): tool_calls: one [], got ['add'] | rand:3:8 (fail): tool_calls: one [], got ['add'] | rand:4:8 (fail): tool_calls: one [], got ['add'] | rand:5:8 (fail): tool_calls: one [], got ['add']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] add: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] add: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Mistral-7B-Instruct-v0.3", "tokenizer": { "repo": "mistralai/Mistral-7B-Instruct-v0.3", "revision": "c170c708c41dac9275d15a8fff4eca08d52bab71", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "MistralForCausalLM" ], "model_type": "mistral" }, "hf_config_error": null, "chat_template_sha256": "e16746b40344d6c5b5265988e0328a0bf7277be86f1c335156eae07e29c82826", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 4 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "Hello", "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "{\"a\": 1, \"b\": 2}" } ], "exception": null }, "streams": { "one": { "content": " Hello", "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": " Hello", "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "" } ], "exception": null }, "token": { "content": " Hello", "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "{\"a\": 1, \"b\": 2}" } ], "exception": null }, "rand:1:8": { "content": " Hello", "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "{\"a\": 1, \"b\": 2}" } ], "exception": null }, "rand:2:8": { "content": " Hello", "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "{\"a\": 1, \"b\": 2}" } ], "exception": null }, "rand:3:8": { "content": " Hello", "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "{\"a\": 1, \"b\": 2}" } ], "exception": null }, "rand:4:8": { "content": " Hello", "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "{\"a\": 1, \"b\": 2}" } ], "exception": null }, "rand:5:8": { "content": " Hello", "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "{\"a\": 1, \"b\": 2}" } ], "exception": null } } } }, { "fixture_id": "mistral/vllm-v3-malformed-not-json", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Mistral-7B-Instruct-v0.3", "tokenizer": { "repo": "mistralai/Mistral-7B-Instruct-v0.3", "revision": "c170c708c41dac9275d15a8fff4eca08d52bab71", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "MistralForCausalLM" ], "model_type": "mistral" }, "hf_config_error": null, "chat_template_sha256": "e16746b40344d6c5b5265988e0328a0bf7277be86f1c335156eae07e29c82826", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 4 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "mistral/vllm-v11-single-call", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Mistral-Small-3.2-24B-Instruct-2506", "tokenizer": { "repo": "mistralai/Mistral-Small-3.2-24B-Instruct-2506", "revision": "95a6d26c4bfb886c58daf9d3f7332c857cb27b43", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "mistral/vllm-v11-parallel", "family": "mistral", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['add', 'multiply'], got ['add']" }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream ['add', 'multiply'], got ['add']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: special (fail): tool_calls: one ['add'], got ['add', 'multiply'] | token (fail): tool_calls: one ['add'], got ['add', 'multiply'] | rand:1:8 (fail): tool_calls: one ['add'], got ['add', 'multiply'] | rand:2:8 (fail): tool_calls: one ['add'], got ['add', 'multiply'] | rand:3:8 (fail): tool_calls: one ['add'], got ['add', 'multiply'] | rand:4:8 (fail): tool_calls: one ['add'], got ['add', 'multiply'] | rand:5:8 (fail): tool_calls: one ['add'], got ['add', 'multiply']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "fail", "detail": "expected 2 calls ['add', 'multiply'], got 1 ['add']" }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Mistral-Small-3.2-24B-Instruct-2506", "tokenizer": { "repo": "mistralai/Mistral-Small-3.2-24B-Instruct-2506", "revision": "95a6d26c4bfb886c58daf9d3f7332c857cb27b43", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "{\"a\": 3.5, \"b\": 4}" }, { "name": "multiply", "arguments_raw": "{\"a\": 3, \"b\": 6}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "{\"a\": 3.5, \"b\": 4}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "{\"a\": 3.5, \"b\": 4}" }, { "name": "multiply", "arguments_raw": "{\"a\": 3, \"b\": 6}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "{\"a\": 3.5, \"b\": 4}" }, { "name": "multiply", "arguments_raw": "{\"a\": 3, \"b\": 6}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "{\"a\": 3.5, \"b\": 4}" }, { "name": "multiply", "arguments_raw": "{\"a\": 3, \"b\": 6}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "{\"a\": 3.5, \"b\": 4}" }, { "name": "multiply", "arguments_raw": "{\"a\": 3, \"b\": 6}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "{\"a\": 3.5, \"b\": 4}" }, { "name": "multiply", "arguments_raw": "{\"a\": 3, \"b\": 6}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "{\"a\": 3.5, \"b\": 4}" }, { "name": "multiply", "arguments_raw": "{\"a\": 3, \"b\": 6}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "{\"a\": 3.5, \"b\": 4}" }, { "name": "multiply", "arguments_raw": "{\"a\": 3, \"b\": 6}" } ], "exception": null } } } }, { "fixture_id": "mistral/vllm-v11-content-before-call", "family": "mistral", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['add'], got []" }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream ['add'], got []" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: special (fail): tool_calls: one [], got ['add'] | token (fail): tool_calls: one [], got ['add'] | rand:1:8 (fail): tool_calls: one [], got ['add'] | rand:2:8 (fail): tool_calls: one [], got ['add'] | rand:3:8 (fail): tool_calls: one [], got ['add'] | rand:4:8 (fail): tool_calls: one [], got ['add'] | rand:5:8 (fail): tool_calls: one [], got ['add']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Mistral-Small-3.2-24B-Instruct-2506", "tokenizer": { "repo": "mistralai/Mistral-Small-3.2-24B-Instruct-2506", "revision": "95a6d26c4bfb886c58daf9d3f7332c857cb27b43", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "hi", "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "{\"a\": 1, \"b\": 2}" } ], "exception": null }, "streams": { "one": { "content": "hi", "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": "hi", "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "{\"a\": 1, \"b\": 2}" } ], "exception": null }, "token": { "content": "hi", "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "{\"a\": 1, \"b\": 2}" } ], "exception": null }, "rand:1:8": { "content": "hi", "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "{\"a\": 1, \"b\": 2}" } ], "exception": null }, "rand:2:8": { "content": "hi", "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "{\"a\": 1, \"b\": 2}" } ], "exception": null }, "rand:3:8": { "content": "hi", "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "{\"a\": 1, \"b\": 2}" } ], "exception": null }, "rand:4:8": { "content": "hi", "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "{\"a\": 1, \"b\": 2}" } ], "exception": null }, "rand:5:8": { "content": "hi", "reasoning_content": null, "tool_calls": [ { "name": "add", "arguments_raw": "{\"a\": 1, \"b\": 2}" } ], "exception": null } } } }, { "fixture_id": "mistral/vllm-v13-think-no-end", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": "mistral", "tool_call_detector": "MistralDetector", "reasoning_detector": "MistralDetector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Magistral-Small-2509", "tokenizer": { "repo": "mistralai/Magistral-Small-2509", "revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": null, "reasoning_effort": "medium", "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "mistral/vllm-v11-think-text", "family": "mistral", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected 'c', got None; reasoning_content: expected 'r', got 'rc'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected 'c', got None; reasoning_content: expected 'r', got 'rc'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected 'c', got None; reasoning_content: expected 'r', got 'rc'" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected 'c', got None; reasoning_content: expected 'r', got 'rc'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected 'c', got None; reasoning_content: expected 'r', got 'rc'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected 'c', got None; reasoning_content: expected 'r', got 'rc'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected 'c', got None; reasoning_content: expected 'r', got 'rc'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected 'c', got None; reasoning_content: expected 'r', got 'rc'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected 'c', got None; reasoning_content: expected 'r', got 'rc'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "reasoning_content contains ''; reasoning_content contains ''" }, { "check": "no_leakage", "strategy": "one", "status": "fail", "detail": "reasoning_content contains ''; reasoning_content contains ''" }, { "check": "no_leakage", "strategy": "special", "status": "fail", "detail": "reasoning_content contains ''; reasoning_content contains ''" }, { "check": "no_leakage", "strategy": "token", "status": "fail", "detail": "reasoning_content contains ''; reasoning_content contains ''" }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "fail", "detail": "reasoning_content contains ''; reasoning_content contains ''" }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "fail", "detail": "reasoning_content contains ''; reasoning_content contains ''" }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "fail", "detail": "reasoning_content contains ''; reasoning_content contains ''" }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "fail", "detail": "reasoning_content contains ''; reasoning_content contains ''" }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "fail", "detail": "reasoning_content contains ''; reasoning_content contains ''" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": "mistral", "tool_call_detector": "MistralDetector", "reasoning_detector": "MistralDetector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Magistral-Small-2506", "tokenizer": { "repo": "mistralai/Magistral-Small-2506", "revision": "ad2fce5b4113139e1877dcadfd826c9262ad8e8c", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "MistralForCausalLM" ], "model_type": "mistral" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": null, "reasoning_effort": "medium", "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "rc", "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "rc", "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": "rc", "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": "rc", "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "rc", "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "rc", "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "rc", "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "rc", "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "rc", "tool_calls": [], "exception": null } } } }, { "fixture_id": "mistral/llamacpp-ministral3-reasoning-call", "family": "mistral", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got \"[THINK]I'm\\nthinking[/THINK]\"; reasoning_content: expected \"I'm\\nthinking\", got None" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected None, got \"[THINK]I'm\\nthinking[/THINK]\"; reasoning_content: expected \"I'm\\nthinking\", got None; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected None, got \"[THINKI'm\\nthinking[/THINK\"; reasoning_content: expected \"I'm\\nthinking\", got None" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected None, got \"[THINKI'm\\nthinking[/THINK\"; reasoning_content: expected \"I'm\\nthinking\", got None" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected None, got \"[THINKI'm\\nthinking[/THINK]\"; reasoning_content: expected \"I'm\\nthinking\", got None" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected None, got \"[THINKI'm\\nthinking[/THINK]\"; reasoning_content: expected \"I'm\\nthinking\", got None" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected None, got \"[THINKI'm\\nthinking[/THINK\"; reasoning_content: expected \"I'm\\nthinking\", got None" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected None, got \"[THINKI'm\\nthinking[/THINK]\"; reasoning_content: expected \"I'm\\nthinking\", got None" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected None, got \"[THINKI'm\\nthinking[/THINK]\"; reasoning_content: expected \"I'm\\nthinking\", got None" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream ['special_function'], got []" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream \"[THINK]I'm\\nthinking[/THINK]\", got \"[THINKI'm\\nthinking[/THINK\"" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream \"[THINK]I'm\\nthinking[/THINK]\", got \"[THINKI'm\\nthinking[/THINK\"" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream \"[THINK]I'm\\nthinking[/THINK]\", got \"[THINKI'm\\nthinking[/THINK]\"" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream \"[THINK]I'm\\nthinking[/THINK]\", got \"[THINKI'm\\nthinking[/THINK]\"" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream \"[THINK]I'm\\nthinking[/THINK]\", got \"[THINKI'm\\nthinking[/THINK\"" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream \"[THINK]I'm\\nthinking[/THINK]\", got \"[THINKI'm\\nthinking[/THINK]\"" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream \"[THINK]I'm\\nthinking[/THINK]\", got \"[THINKI'm\\nthinking[/THINK]\"" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: special (fail): content: one \"[THINK]I'm\\nthinking[/THINK]\", got \"[THINKI'm\\nthinking[/THINK\"; tool_calls: one [], got ['special_function'] | token (fail): content: one \"[THINK]I'm\\nthinking[/THINK]\", got \"[THINKI'm\\nthinking[/THINK\"; tool_calls: one [], got ['special_function'] | rand:1:8 (fail): content: one \"[THINK]I'm\\nthinking[/THINK]\", got \"[THINKI'm\\nthinking[/THINK]\"; tool_calls: one [], got ['special_function'] | rand:2:8 (fail): content: one \"[THINK]I'm\\nthinking[/THINK]\", got \"[THINKI'm\\nthinking[/THINK]\"; tool_calls: one [], got ['special_function'] | rand:3:8 (fail): content: one \"[THINK]I'm\\nthinking[/THINK]\", got \"[THINKI'm\\nthinking[/THINK\"; tool_calls: one [], got ['special_function'] | rand:4:8 (fail): content: one \"[THINK]I'm\\nthinking[/THINK]\", got \"[THINKI'm\\nthinking[/THINK]\"; tool_calls: one [], got ['special_function'] | rand:5:8 (fail): content: one \"[THINK]I'm\\nthinking[/THINK]\", got \"[THINKI'm\\nthinking[/THINK]\"; tool_calls: one [], got ['special_function']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '[THINK]'; content contains '[/THINK]'" }, { "check": "no_leakage", "strategy": "one", "status": "fail", "detail": "content contains '[THINK]'; content contains '[/THINK]'" }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "fail", "detail": "content contains '[/THINK]'" }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "fail", "detail": "content contains '[/THINK]'" }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "fail", "detail": "content contains '[/THINK]'" }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "fail", "detail": "content contains '[/THINK]'" }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Ministral-3-14B-Reasoning-2512", "tokenizer": { "repo": "mistralai/Ministral-3-14B-Reasoning-2512", "revision": "51f9210f3cd20f3452a80d5819d15dc61cc50630", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [ "thinking=True not applied: Anthropic thinking is not supported for models without a reasoning parser" ] }, "observed": { "nonstream": { "content": "[THINK]I'm\nthinking[/THINK]", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "streams": { "one": { "content": "[THINK]I'm\nthinking[/THINK]", "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": "[THINKI'm\nthinking[/THINK", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "token": { "content": "[THINKI'm\nthinking[/THINK", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "rand:1:8": { "content": "[THINKI'm\nthinking[/THINK]", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "rand:2:8": { "content": "[THINKI'm\nthinking[/THINK]", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "rand:3:8": { "content": "[THINKI'm\nthinking[/THINK", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "rand:4:8": { "content": "[THINKI'm\nthinking[/THINK]", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "rand:5:8": { "content": "[THINKI'm\nthinking[/THINK]", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null } } } }, { "fixture_id": "mistral/llamacpp-ministral3-parallel", "family": "mistral", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['special_function', 'special_function_with_opt'], got ['special_function']" }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream ['special_function', 'special_function_with_opt'], got ['special_function']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: special (fail): tool_calls: one ['special_function'], got ['special_function', 'special_function_with_opt'] | token (fail): tool_calls: one ['special_function'], got ['special_function', 'special_function_with_opt'] | rand:1:8 (fail): tool_calls: one ['special_function'], got ['special_function', 'special_function_with_opt'] | rand:2:8 (fail): tool_calls: one ['special_function'], got ['special_function', 'special_function_with_opt'] | rand:3:8 (fail): tool_calls: one ['special_function'], got ['special_function', 'special_function_with_opt'] | rand:4:8 (fail): tool_calls: one ['special_function'], got ['special_function', 'special_function_with_opt'] | rand:5:8 (fail): tool_calls: one ['special_function'], got ['special_function', 'special_function_with_opt']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "fail", "detail": "expected 2 calls ['special_function', 'special_function_with_opt'], got 1 ['special_function']" }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Ministral-3-14B-Reasoning-2512", "tokenizer": { "repo": "mistralai/Ministral-3-14B-Reasoning-2512", "revision": "51f9210f3cd20f3452a80d5819d15dc61cc50630", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [ "thinking=True not applied: Anthropic thinking is not supported for models without a reasoning parser" ] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\": 1, \"arg2\": 2}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\": 1, \"arg2\": 2}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\": 1, \"arg2\": 2}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\": 1, \"arg2\": 2}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\": 1, \"arg2\": 2}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\": 1, \"arg2\": 2}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\": 1, \"arg2\": 2}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\": 1, \"arg2\": 2}" } ], "exception": null } } } }, { "fixture_id": "mistral/llamacpp-ministral3-marker-in-reasoning", "family": "mistral", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '[THINK]Let me think about'; reasoning_content: expected 'Let me think about [TOOL_CALLS]special_function[ARGS]{\"arg1\":1} and more', got None; tool_calls: expected ['special_function'], got ['special_function', 'special_function']" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected None, got '[THINK]Let me think about '; reasoning_content: expected 'Let me think about [TOOL_CALLS]special_function[ARGS]{\"arg1\":1} and more', got None; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected None, got '[THINKLet me think about and more[/THINK'; reasoning_content: expected 'Let me think about [TOOL_CALLS]special_function[ARGS]{\"arg1\":1} and more', got None; tool_calls: expected ['special_function'], got ['special_function', 'special_function']" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected None, got '[THINKLet me think about and more[/THINK'; reasoning_content: expected 'Let me think about [TOOL_CALLS]special_function[ARGS]{\"arg1\":1} and more', got None; tool_calls: expected ['special_function'], got ['special_function', 'special_function']" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected None, got '[THINKLet me think about and more[/THINK]'; reasoning_content: expected 'Let me think about [TOOL_CALLS]special_function[ARGS]{\"arg1\":1} and more', got None; tool_calls: expected ['special_function'], got ['special_function', 'special_function']" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected None, got '[THINKLet me think about and more[/THINK]'; reasoning_content: expected 'Let me think about [TOOL_CALLS]special_function[ARGS]{\"arg1\":1} and more', got None; tool_calls: expected ['special_function'], got ['special_function', 'special_function']" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected None, got '[THINKLet me think about and more[/THINK]'; reasoning_content: expected 'Let me think about [TOOL_CALLS]special_function[ARGS]{\"arg1\":1} and more', got None; tool_calls: expected ['special_function'], got ['special_function', 'special_function']" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected None, got '[THINKLet me think about and more[/THINK]'; reasoning_content: expected 'Let me think about [TOOL_CALLS]special_function[ARGS]{\"arg1\":1} and more', got None; tool_calls: expected ['special_function'], got ['special_function', 'special_function']" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected None, got '[THINKLet me think about and more[/THINK]'; reasoning_content: expected 'Let me think about [TOOL_CALLS]special_function[ARGS]{\"arg1\":1} and more', got None; tool_calls: expected ['special_function'], got ['special_function', 'special_function']" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content (whitespace only): nonstream '[THINK]Let me think about', got '[THINK]Let me think about '; tool_calls: nonstream ['special_function', 'special_function'], got []" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '[THINK]Let me think about', got '[THINKLet me think about and more[/THINK'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '[THINK]Let me think about', got '[THINKLet me think about and more[/THINK'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '[THINK]Let me think about', got '[THINKLet me think about and more[/THINK]'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '[THINK]Let me think about', got '[THINKLet me think about and more[/THINK]'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '[THINK]Let me think about', got '[THINKLet me think about and more[/THINK]'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '[THINK]Let me think about', got '[THINKLet me think about and more[/THINK]'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '[THINK]Let me think about', got '[THINKLet me think about and more[/THINK]'" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: special (fail): content: one '[THINK]Let me think about ', got '[THINKLet me think about and more[/THINK'; tool_calls: one [], got ['special_function', 'special_function'] | token (fail): content: one '[THINK]Let me think about ', got '[THINKLet me think about and more[/THINK'; tool_calls: one [], got ['special_function', 'special_function'] | rand:1:8 (fail): content: one '[THINK]Let me think about ', got '[THINKLet me think about and more[/THINK]'; tool_calls: one [], got ['special_function', 'special_function'] | rand:2:8 (fail): content: one '[THINK]Let me think about ', got '[THINKLet me think about and more[/THINK]'; tool_calls: one [], got ['special_function', 'special_function'] | rand:3:8 (fail): content: one '[THINK]Let me think about ', got '[THINKLet me think about and more[/THINK]'; tool_calls: one [], got ['special_function', 'special_function'] | rand:4:8 (fail): content: one '[THINK]Let me think about ', got '[THINKLet me think about and more[/THINK]'; tool_calls: one [], got ['special_function', 'special_function'] | rand:5:8 (fail): content: one '[THINK]Let me think about ', got '[THINKLet me think about and more[/THINK]'; tool_calls: one [], got ['special_function', 'special_function']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains '[THINK]'" }, { "check": "no_leakage", "strategy": "one", "status": "fail", "detail": "content contains '[THINK]'" }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "fail", "detail": "content contains '[/THINK]'" }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "fail", "detail": "content contains '[/THINK]'" }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "fail", "detail": "content contains '[/THINK]'" }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "fail", "detail": "content contains '[/THINK]'" }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "fail", "detail": "content contains '[/THINK]'" }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Ministral-3-14B-Reasoning-2512", "tokenizer": { "repo": "mistralai/Ministral-3-14B-Reasoning-2512", "revision": "51f9210f3cd20f3452a80d5819d15dc61cc50630", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [ "thinking=True not applied: Anthropic thinking is not supported for models without a reasoning parser" ] }, "observed": { "nonstream": { "content": "[THINK]Let me think about", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "streams": { "one": { "content": "[THINK]Let me think about ", "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": "[THINKLet me think about and more[/THINK", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "token": { "content": "[THINKLet me think about and more[/THINK", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "rand:1:8": { "content": "[THINKLet me think about and more[/THINK]", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "rand:2:8": { "content": "[THINKLet me think about and more[/THINK]", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "rand:3:8": { "content": "[THINKLet me think about and more[/THINK]", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "rand:4:8": { "content": "[THINKLet me think about and more[/THINK]", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "rand:5:8": { "content": "[THINKLet me think about and more[/THINK]", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null } } } }, { "fixture_id": "mistral/llamacpp-v11-call-id", "family": "mistral", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected None, got '[TOOL_CALLS]special_function[CALL_ID]123456789[ARGS]{\"arg1\": 1}'; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected None, got '[TOOL_CALLS]special_function[CALL_ID]123456789[ARGS]{\"arg1\": 1}'; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected None, got '[TOOL_CALLS]special_function[CALL_ID]123456789[ARGS]{\"arg1\": 1}'; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected None, got '[TOOL_CALLS]special_function[CALL_ID]123456789[ARGS]{\"arg1\": 1}'; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected None, got '[TOOL_CALLS]special_function[CALL_ID]123456789[ARGS]{\"arg1\": 1}'; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected None, got '[TOOL_CALLS]special_function[CALL_ID]123456789[ARGS]{\"arg1\": 1}'; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected None, got '[TOOL_CALLS]special_function[CALL_ID]123456789[ARGS]{\"arg1\": 1}'; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected None, got '[TOOL_CALLS]special_function[CALL_ID]123456789[ARGS]{\"arg1\": 1}'; tool_calls: expected ['special_function'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream None, got '[TOOL_CALLS]special_function[CALL_ID]123456789[ARGS]{\"arg1\": 1}'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream None, got '[TOOL_CALLS]special_function[CALL_ID]123456789[ARGS]{\"arg1\": 1}'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream None, got '[TOOL_CALLS]special_function[CALL_ID]123456789[ARGS]{\"arg1\": 1}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream None, got '[TOOL_CALLS]special_function[CALL_ID]123456789[ARGS]{\"arg1\": 1}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream None, got '[TOOL_CALLS]special_function[CALL_ID]123456789[ARGS]{\"arg1\": 1}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream None, got '[TOOL_CALLS]special_function[CALL_ID]123456789[ARGS]{\"arg1\": 1}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream None, got '[TOOL_CALLS]special_function[CALL_ID]123456789[ARGS]{\"arg1\": 1}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream None, got '[TOOL_CALLS]special_function[CALL_ID]123456789[ARGS]{\"arg1\": 1}'" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "fail", "detail": "content contains '[TOOL_CALLS]'; content contains '[ARGS]'; content contains '[CALL_ID]'" }, { "check": "no_leakage", "strategy": "special", "status": "fail", "detail": "content contains '[TOOL_CALLS]'; content contains '[ARGS]'; content contains '[CALL_ID]'" }, { "check": "no_leakage", "strategy": "token", "status": "fail", "detail": "content contains '[TOOL_CALLS]'; content contains '[ARGS]'; content contains '[CALL_ID]'" }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "fail", "detail": "content contains '[TOOL_CALLS]'; content contains '[ARGS]'; content contains '[CALL_ID]'" }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "fail", "detail": "content contains '[TOOL_CALLS]'; content contains '[ARGS]'; content contains '[CALL_ID]'" }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "fail", "detail": "content contains '[TOOL_CALLS]'; content contains '[ARGS]'; content contains '[CALL_ID]'" }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "fail", "detail": "content contains '[TOOL_CALLS]'; content contains '[ARGS]'; content contains '[CALL_ID]'" }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "fail", "detail": "content contains '[TOOL_CALLS]'; content contains '[ARGS]'; content contains '[CALL_ID]'" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Mistral-Small-3.2-24B-Instruct-2506", "tokenizer": { "repo": "mistralai/Mistral-Small-3.2-24B-Instruct-2506", "revision": "95a6d26c4bfb886c58daf9d3f7332c857cb27b43", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": "[TOOL_CALLS]special_function[CALL_ID]123456789[ARGS]{\"arg1\": 1}", "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": "[TOOL_CALLS]special_function[CALL_ID]123456789[ARGS]{\"arg1\": 1}", "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": "[TOOL_CALLS]special_function[CALL_ID]123456789[ARGS]{\"arg1\": 1}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:1:8": { "content": "[TOOL_CALLS]special_function[CALL_ID]123456789[ARGS]{\"arg1\": 1}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:2:8": { "content": "[TOOL_CALLS]special_function[CALL_ID]123456789[ARGS]{\"arg1\": 1}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:3:8": { "content": "[TOOL_CALLS]special_function[CALL_ID]123456789[ARGS]{\"arg1\": 1}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:4:8": { "content": "[TOOL_CALLS]special_function[CALL_ID]123456789[ARGS]{\"arg1\": 1}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:5:8": { "content": "[TOOL_CALLS]special_function[CALL_ID]123456789[ARGS]{\"arg1\": 1}", "reasoning_content": null, "tool_calls": [], "exception": null } } } }, { "fixture_id": "mistral/llamacpp-v11-call-id-parallel", "family": "mistral", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "tool_calls: expected ['special_function', 'special_function_with_opt'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected None, got '[TOOL_CALLS]special_function[CALL_ID]000000001[ARGS]{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt[CALL_ID]000000002[ARGS]{\"arg1\": 1, \"arg2\": 2}'; tool_calls: expected ['special_function', 'special_function_with_opt'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected None, got '[TOOL_CALLS]special_function[CALL_ID]000000001[ARGS]{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt[CALL_ID]000000002[ARGS]{\"arg1\": 1, \"arg2\": 2}'; tool_calls: expected ['special_function', 'special_function_with_opt'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected None, got '[TOOL_CALLS]special_function[CALL_ID]000000001[ARGS]{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt[CALL_ID]000000002[ARGS]{\"arg1\": 1, \"arg2\": 2}'; tool_calls: expected ['special_function', 'special_function_with_opt'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected None, got '[TOOL_CALLS]special_function[CALL_ID]000000001[ARGS]{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt[CALL_ID]000000002[ARGS{\"arg1\": 1, \"arg2\": 2}'; tool_calls: expected ['special_function', 'special_function_with_opt'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected None, got '[TOOL_CALLS]special_function[CALL_ID]000000001[ARGS]{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt[CALL_ID000000002[ARGS{\"arg1\": 1, \"arg2\": 2}'; tool_calls: expected ['special_function', 'special_function_with_opt'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected None, got '[TOOL_CALLS]special_function[CALL_ID]000000001[ARGS]{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt[CALL_ID]000000002[ARGS]{\"arg1\": 1, \"arg2\": 2}'; tool_calls: expected ['special_function', 'special_function_with_opt'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected None, got '[TOOL_CALLS]special_function[CALL_ID]000000001[ARGS]{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt[CALL_ID000000002[ARGS{\"arg1\": 1, \"arg2\": 2}'; tool_calls: expected ['special_function', 'special_function_with_opt'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected None, got '[TOOL_CALLS]special_function[CALL_ID]000000001[ARGS]{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt[CALL_ID000000002[ARGS{\"arg1\": 1, \"arg2\": 2}'; tool_calls: expected ['special_function', 'special_function_with_opt'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream None, got '[TOOL_CALLS]special_function[CALL_ID]000000001[ARGS]{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt[CALL_ID]000000002[ARGS]{\"arg1\": 1, \"arg2\": 2}'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream None, got '[TOOL_CALLS]special_function[CALL_ID]000000001[ARGS]{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt[CALL_ID]000000002[ARGS]{\"arg1\": 1, \"arg2\": 2}'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream None, got '[TOOL_CALLS]special_function[CALL_ID]000000001[ARGS]{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt[CALL_ID]000000002[ARGS]{\"arg1\": 1, \"arg2\": 2}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream None, got '[TOOL_CALLS]special_function[CALL_ID]000000001[ARGS]{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt[CALL_ID]000000002[ARGS{\"arg1\": 1, \"arg2\": 2}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream None, got '[TOOL_CALLS]special_function[CALL_ID]000000001[ARGS]{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt[CALL_ID000000002[ARGS{\"arg1\": 1, \"arg2\": 2}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream None, got '[TOOL_CALLS]special_function[CALL_ID]000000001[ARGS]{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt[CALL_ID]000000002[ARGS]{\"arg1\": 1, \"arg2\": 2}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream None, got '[TOOL_CALLS]special_function[CALL_ID]000000001[ARGS]{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt[CALL_ID000000002[ARGS{\"arg1\": 1, \"arg2\": 2}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream None, got '[TOOL_CALLS]special_function[CALL_ID]000000001[ARGS]{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt[CALL_ID000000002[ARGS{\"arg1\": 1, \"arg2\": 2}'" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: rand:1:8 (fail): content: one '[TOOL_CALLS]special_function[CALL_ID]000000001[ARGS]{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt[CALL_ID]000000002[ARGS]{\"arg1\": 1, \"arg2\": 2}', got '[TOOL_CALLS]special_function[CALL_ID]000000001[ARGS]{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt[CALL_ID]000000002[ARGS{\"arg1\": 1, \"arg2\": 2}' | rand:2:8 (fail): content: one '[TOOL_CALLS]special_function[CALL_ID]000000001[ARGS]{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt[CALL_ID]000000002[ARGS]{\"arg1\": 1, \"arg2\": 2}', got '[TOOL_CALLS]special_function[CALL_ID]000000001[ARGS]{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt[CALL_ID000000002[ARGS{\"arg1\": 1, \"arg2\": 2}' | rand:4:8 (fail): content: one '[TOOL_CALLS]special_function[CALL_ID]000000001[ARGS]{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt[CALL_ID]000000002[ARGS]{\"arg1\": 1, \"arg2\": 2}', got '[TOOL_CALLS]special_function[CALL_ID]000000001[ARGS]{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt[CALL_ID000000002[ARGS{\"arg1\": 1, \"arg2\": 2}' | rand:5:8 (fail): content: one '[TOOL_CALLS]special_function[CALL_ID]000000001[ARGS]{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt[CALL_ID]000000002[ARGS]{\"arg1\": 1, \"arg2\": 2}', got '[TOOL_CALLS]special_function[CALL_ID]000000001[ARGS]{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt[CALL_ID000000002[ARGS{\"arg1\": 1, \"arg2\": 2}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "fail", "detail": "content contains '[TOOL_CALLS]'; content contains '[ARGS]'; content contains '[CALL_ID]'" }, { "check": "no_leakage", "strategy": "special", "status": "fail", "detail": "content contains '[TOOL_CALLS]'; content contains '[ARGS]'; content contains '[CALL_ID]'" }, { "check": "no_leakage", "strategy": "token", "status": "fail", "detail": "content contains '[TOOL_CALLS]'; content contains '[ARGS]'; content contains '[CALL_ID]'" }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "fail", "detail": "content contains '[TOOL_CALLS]'; content contains '[ARGS]'; content contains '[CALL_ID]'" }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "fail", "detail": "content contains '[TOOL_CALLS]'; content contains '[ARGS]'; content contains '[CALL_ID]'" }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "fail", "detail": "content contains '[TOOL_CALLS]'; content contains '[ARGS]'; content contains '[CALL_ID]'" }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "fail", "detail": "content contains '[TOOL_CALLS]'; content contains '[ARGS]'; content contains '[CALL_ID]'" }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "fail", "detail": "content contains '[TOOL_CALLS]'; content contains '[ARGS]'; content contains '[CALL_ID]'" }, { "check": "parallel_order", "strategy": "nonstream", "status": "fail", "detail": "expected 2 calls ['special_function', 'special_function_with_opt'], got 0 []" }, { "check": "parallel_order", "strategy": "one", "status": "fail", "detail": "expected 2 calls ['special_function', 'special_function_with_opt'], got 0 []" }, { "check": "parallel_order", "strategy": "special", "status": "fail", "detail": "expected 2 calls ['special_function', 'special_function_with_opt'], got 0 []" }, { "check": "parallel_order", "strategy": "token", "status": "fail", "detail": "expected 2 calls ['special_function', 'special_function_with_opt'], got 0 []" }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "fail", "detail": "expected 2 calls ['special_function', 'special_function_with_opt'], got 0 []" }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "fail", "detail": "expected 2 calls ['special_function', 'special_function_with_opt'], got 0 []" }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "fail", "detail": "expected 2 calls ['special_function', 'special_function_with_opt'], got 0 []" }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "fail", "detail": "expected 2 calls ['special_function', 'special_function_with_opt'], got 0 []" }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "fail", "detail": "expected 2 calls ['special_function', 'special_function_with_opt'], got 0 []" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Mistral-Small-3.2-24B-Instruct-2506", "tokenizer": { "repo": "mistralai/Mistral-Small-3.2-24B-Instruct-2506", "revision": "95a6d26c4bfb886c58daf9d3f7332c857cb27b43", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": "[TOOL_CALLS]special_function[CALL_ID]000000001[ARGS]{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt[CALL_ID]000000002[ARGS]{\"arg1\": 1, \"arg2\": 2}", "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": "[TOOL_CALLS]special_function[CALL_ID]000000001[ARGS]{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt[CALL_ID]000000002[ARGS]{\"arg1\": 1, \"arg2\": 2}", "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": "[TOOL_CALLS]special_function[CALL_ID]000000001[ARGS]{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt[CALL_ID]000000002[ARGS]{\"arg1\": 1, \"arg2\": 2}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:1:8": { "content": "[TOOL_CALLS]special_function[CALL_ID]000000001[ARGS]{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt[CALL_ID]000000002[ARGS{\"arg1\": 1, \"arg2\": 2}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:2:8": { "content": "[TOOL_CALLS]special_function[CALL_ID]000000001[ARGS]{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt[CALL_ID000000002[ARGS{\"arg1\": 1, \"arg2\": 2}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:3:8": { "content": "[TOOL_CALLS]special_function[CALL_ID]000000001[ARGS]{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt[CALL_ID]000000002[ARGS]{\"arg1\": 1, \"arg2\": 2}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:4:8": { "content": "[TOOL_CALLS]special_function[CALL_ID]000000001[ARGS]{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt[CALL_ID000000002[ARGS{\"arg1\": 1, \"arg2\": 2}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:5:8": { "content": "[TOOL_CALLS]special_function[CALL_ID]000000001[ARGS]{\"arg1\": 1}[TOOL_CALLS]special_function_with_opt[CALL_ID000000002[ARGS{\"arg1\": 1, \"arg2\": 2}", "reasoning_content": null, "tool_calls": [], "exception": null } } } }, { "fixture_id": "mistral/llamacpp-devstral-content-before-call", "family": "mistral", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream ['special_function'], got []" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: special (fail): tool_calls: one [], got ['special_function'] | token (fail): tool_calls: one [], got ['special_function'] | rand:1:8 (fail): tool_calls: one [], got ['special_function'] | rand:2:8 (fail): tool_calls: one [], got ['special_function'] | rand:3:8 (fail): tool_calls: one [], got ['special_function'] | rand:4:8 (fail): tool_calls: one [], got ['special_function'] | rand:5:8 (fail): tool_calls: one [], got ['special_function']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Devstral-Small-2507", "tokenizer": { "repo": "mistralai/Devstral-Small-2507", "revision": "bd165ab26cebbcc2eea2c4ecbfc07f3ac42b3c39", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "MistralForCausalLM" ], "model_type": "mistral" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "Hello, world!\nWhat's up?", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "streams": { "one": { "content": "Hello, world!\nWhat's up?", "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": "Hello, world!\nWhat's up?", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "token": { "content": "Hello, world!\nWhat's up?", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "rand:1:8": { "content": "Hello, world!\nWhat's up?", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "rand:2:8": { "content": "Hello, world!\nWhat's up?", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "rand:3:8": { "content": "Hello, world!\nWhat's up?", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "rand:4:8": { "content": "Hello, world!\nWhat's up?", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "rand:5:8": { "content": "Hello, world!\nWhat's up?", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null } } } }, { "fixture_id": "mistral/bug-devstral-glob-braces-in-arguments", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Devstral-Small-2507", "tokenizer": { "repo": "mistralai/Devstral-Small-2507", "revision": "bd165ab26cebbcc2eea2c4ecbfc07f3ac42b3c39", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "MistralForCausalLM" ], "model_type": "mistral" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "mistral/v13think-single-call", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": "mistral", "tool_call_detector": "MistralDetector", "reasoning_detector": "MistralDetector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Magistral-Small-2509", "tokenizer": { "repo": "mistralai/Magistral-Small-2509", "revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": null, "reasoning_effort": "medium", "reasoning_enabled": true, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "mistral/v13think-content-before-call", "family": "mistral", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream ['get_weather'], got []" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: special (fail): tool_calls: one [], got ['get_weather'] | token (fail): tool_calls: one [], got ['get_weather'] | rand:1:8 (fail): tool_calls: one [], got ['get_weather'] | rand:2:8 (fail): tool_calls: one [], got ['get_weather'] | rand:3:8 (fail): tool_calls: one [], got ['get_weather'] | rand:4:8 (fail): tool_calls: one [], got ['get_weather'] | rand:5:8 (fail): tool_calls: one [], got ['get_weather']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": "mistral", "tool_call_detector": "MistralDetector", "reasoning_detector": "MistralDetector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Magistral-Small-2509", "tokenizer": { "repo": "mistralai/Magistral-Small-2509", "revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": null, "reasoning_effort": "medium", "reasoning_enabled": true, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "Let me check that for you.", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "streams": { "one": { "content": "Let me check that for you.", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.", "tool_calls": [], "exception": null }, "special": { "content": "Let me check that for you.", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "token": { "content": "Let me check that for you.", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "rand:1:8": { "content": "Let me check that for you.", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "rand:2:8": { "content": "Let me check that for you.", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "rand:3:8": { "content": "Let me check that for you.", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "rand:4:8": { "content": "Let me check that for you.", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "rand:5:8": { "content": "Let me check that for you.", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null } } } }, { "fixture_id": "mistral/v13think-parallel-calls", "family": "mistral", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'search'], got ['get_weather']" }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'search'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: special (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search'] | token (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search'] | rand:1:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search'] | rand:2:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search'] | rand:3:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search'] | rand:4:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search'] | rand:5:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "fail", "detail": "expected 2 calls ['get_weather', 'search'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": "mistral", "tool_call_detector": "MistralDetector", "reasoning_detector": "MistralDetector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Magistral-Small-2509", "tokenizer": { "repo": "mistralai/Magistral-Small-2509", "revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": null, "reasoning_effort": "medium", "reasoning_enabled": true, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "I should call the tools.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "I should call the tools.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "I should call the tools.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "token": { "content": null, "reasoning_content": "I should call the tools.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "I should call the tools.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "I should call the tools.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "I should call the tools.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "I should call the tools.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "I should call the tools.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null } } } }, { "fixture_id": "mistral/v13think-no-call", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": "mistral", "tool_call_detector": "MistralDetector", "reasoning_detector": "MistralDetector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Magistral-Small-2509", "tokenizer": { "repo": "mistralai/Magistral-Small-2509", "revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": null, "reasoning_effort": "medium", "reasoning_enabled": true, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "mistral/v13think-empty-arguments", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": "mistral", "tool_call_detector": "MistralDetector", "reasoning_detector": "MistralDetector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Magistral-Small-2509", "tokenizer": { "repo": "mistralai/Magistral-Small-2509", "revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": null, "reasoning_effort": "medium", "reasoning_enabled": true, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "mistral/v13think-marker-in-arguments", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": "mistral", "tool_call_detector": "MistralDetector", "reasoning_detector": "MistralDetector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Magistral-Small-2509", "tokenizer": { "repo": "mistralai/Magistral-Small-2509", "revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": null, "reasoning_effort": "medium", "reasoning_enabled": true, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "mistral/v13think-multiline-reasoning", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": "mistral", "tool_call_detector": "MistralDetector", "reasoning_detector": "MistralDetector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Magistral-Small-2509", "tokenizer": { "repo": "mistralai/Magistral-Small-2509", "revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": null, "reasoning_effort": "medium", "reasoning_enabled": true, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "mistral/v13think-truncated-in-reasoning", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": "mistral", "tool_call_detector": "MistralDetector", "reasoning_detector": "MistralDetector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Magistral-Small-2509", "tokenizer": { "repo": "mistralai/Magistral-Small-2509", "revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": null, "reasoning_effort": "medium", "reasoning_enabled": true, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "mistral/v13think-truncated-in-arguments", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": "mistral", "tool_call_detector": "MistralDetector", "reasoning_detector": "MistralDetector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Magistral-Small-2509", "tokenizer": { "repo": "mistralai/Magistral-Small-2509", "revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": null, "reasoning_effort": "medium", "reasoning_enabled": true, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "mistral/v13think-stop-at-open-marker", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": "mistral", "tool_call_detector": "MistralDetector", "reasoning_detector": "MistralDetector", "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Magistral-Small-2509", "tokenizer": { "repo": "mistralai/Magistral-Small-2509", "revision": "a31cc96ab10cf19bc42c628fedf1e359e0853c49", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": null, "reasoning_effort": "medium", "reasoning_enabled": true, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "mistral/v13-single-call", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Ministral-3-14B-Instruct-2512", "tokenizer": { "repo": "mistralai/Ministral-3-14B-Instruct-2512", "revision": "29439f81c2be264d8d393273f99e7db9c0961120", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "mistral/v13-parallel-calls", "family": "mistral", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_weather', 'get_time'], got ['get_weather']" }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_weather', 'get_time'], got ['get_weather', 'get_weather']" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'get_weather', 'get_time'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'get_weather', 'get_time'], got ['get_weather', 'get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: special (fail): tool_calls: one ['get_weather'], got ['get_weather', 'get_weather', 'get_time'] | token (fail): tool_calls: one ['get_weather'], got ['get_weather', 'get_weather', 'get_time'] | rand:1:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'get_weather'] | rand:2:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'get_weather', 'get_time'] | rand:3:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'get_weather', 'get_time'] | rand:4:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'get_weather', 'get_time'] | rand:5:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'get_weather', 'get_time']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "fail", "detail": "expected 3 calls ['get_weather', 'get_weather', 'get_time'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "fail", "detail": "expected 3 calls ['get_weather', 'get_weather', 'get_time'], got 2 ['get_weather', 'get_weather']" }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Ministral-3-14B-Instruct-2512", "tokenizer": { "repo": "mistralai/Ministral-3-14B-Instruct-2512", "revision": "29439f81c2be264d8d393273f99e7db9c0961120", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\"}" }, { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\"}" }, { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\"}" }, { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\"}" }, { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\"}" }, { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\"}" }, { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\"}" }, { "name": "get_time", "arguments_raw": "{}" } ], "exception": null } } } }, { "fixture_id": "mistral/v13-content-before-call", "family": "mistral", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['search'], got []" }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream ['search'], got []" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: special (fail): tool_calls: one [], got ['search'] | token (fail): tool_calls: one [], got ['search'] | rand:1:8 (fail): tool_calls: one [], got ['search'] | rand:2:8 (fail): tool_calls: one [], got ['search'] | rand:3:8 (fail): tool_calls: one [], got ['search'] | rand:4:8 (fail): tool_calls: one [], got ['search'] | rand:5:8 (fail): tool_calls: one [], got ['search']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Ministral-3-14B-Instruct-2512", "tokenizer": { "repo": "mistralai/Ministral-3-14B-Instruct-2512", "revision": "29439f81c2be264d8d393273f99e7db9c0961120", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "I'll search for that.", "reasoning_content": null, "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"vLLM release notes\"}" } ], "exception": null }, "streams": { "one": { "content": "I'll search for that.", "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": "I'll search for that.", "reasoning_content": null, "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"vLLM release notes\"}" } ], "exception": null }, "token": { "content": "I'll search for that.", "reasoning_content": null, "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"vLLM release notes\"}" } ], "exception": null }, "rand:1:8": { "content": "I'll search for that.", "reasoning_content": null, "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"vLLM release notes\"}" } ], "exception": null }, "rand:2:8": { "content": "I'll search for that.", "reasoning_content": null, "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"vLLM release notes\"}" } ], "exception": null }, "rand:3:8": { "content": "I'll search for that.", "reasoning_content": null, "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"vLLM release notes\"}" } ], "exception": null }, "rand:4:8": { "content": "I'll search for that.", "reasoning_content": null, "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"vLLM release notes\"}" } ], "exception": null }, "rand:5:8": { "content": "I'll search for that.", "reasoning_content": null, "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"vLLM release notes\"}" } ], "exception": null } } } }, { "fixture_id": "mistral/v13-no-call", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Ministral-3-14B-Instruct-2512", "tokenizer": { "repo": "mistralai/Ministral-3-14B-Instruct-2512", "revision": "29439f81c2be264d8d393273f99e7db9c0961120", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "mistral/v13-nested-json", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Ministral-3-14B-Instruct-2512", "tokenizer": { "repo": "mistralai/Ministral-3-14B-Instruct-2512", "revision": "29439f81c2be264d8d393273f99e7db9c0961120", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "mistral/v13-unicode-emoji", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Ministral-3-14B-Instruct-2512", "tokenizer": { "repo": "mistralai/Ministral-3-14B-Instruct-2512", "revision": "29439f81c2be264d8d393273f99e7db9c0961120", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "mistral/v13-string-escapes", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Ministral-3-14B-Instruct-2512", "tokenizer": { "repo": "mistralai/Ministral-3-14B-Instruct-2512", "revision": "29439f81c2be264d8d393273f99e7db9c0961120", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "mistral/v13-numeric-arguments", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Ministral-3-14B-Instruct-2512", "tokenizer": { "repo": "mistralai/Ministral-3-14B-Instruct-2512", "revision": "29439f81c2be264d8d393273f99e7db9c0961120", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "mistral/v13-long-arguments", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Ministral-3-14B-Instruct-2512", "tokenizer": { "repo": "mistralai/Ministral-3-14B-Instruct-2512", "revision": "29439f81c2be264d8d393273f99e7db9c0961120", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "mistral/v13-param-named-name", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Ministral-3-14B-Instruct-2512", "tokenizer": { "repo": "mistralai/Ministral-3-14B-Instruct-2512", "revision": "29439f81c2be264d8d393273f99e7db9c0961120", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "mistral/v13-array-lookalike-argument", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Ministral-3-14B-Instruct-2512", "tokenizer": { "repo": "mistralai/Ministral-3-14B-Instruct-2512", "revision": "29439f81c2be264d8d393273f99e7db9c0961120", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "mistral/v11-single-call", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Mistral-Small-3.2-24B-Instruct-2506", "tokenizer": { "repo": "mistralai/Mistral-Small-3.2-24B-Instruct-2506", "revision": "95a6d26c4bfb886c58daf9d3f7332c857cb27b43", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "mistral/v11-parallel-calls", "family": "mistral", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'search'], got ['get_weather']" }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'search'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: special (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search'] | token (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search'] | rand:1:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search'] | rand:2:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search'] | rand:3:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search'] | rand:4:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search'] | rand:5:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "fail", "detail": "expected 2 calls ['get_weather', 'search'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Mistral-Small-3.2-24B-Instruct-2506", "tokenizer": { "repo": "mistralai/Mistral-Small-3.2-24B-Instruct-2506", "revision": "95a6d26c4bfb886c58daf9d3f7332c857cb27b43", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"max\": 3}}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"max\": 3}}" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"max\": 3}}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"max\": 3}}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"max\": 3}}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"max\": 3}}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"max\": 3}}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"max\": 3}}" } ], "exception": null } } } }, { "fixture_id": "mistral/v11-content-before-call", "family": "mistral", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_time'], got []" }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream ['get_time'], got []" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: special (fail): tool_calls: one [], got ['get_time'] | token (fail): tool_calls: one [], got ['get_time'] | rand:1:8 (fail): tool_calls: one [], got ['get_time'] | rand:2:8 (fail): tool_calls: one [], got ['get_time'] | rand:3:8 (fail): tool_calls: one [], got ['get_time'] | rand:4:8 (fail): tool_calls: one [], got ['get_time'] | rand:5:8 (fail): tool_calls: one [], got ['get_time']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Mistral-Small-3.2-24B-Instruct-2506", "tokenizer": { "repo": "mistralai/Mistral-Small-3.2-24B-Instruct-2506", "revision": "95a6d26c4bfb886c58daf9d3f7332c857cb27b43", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "Checking the clock.", "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "streams": { "one": { "content": "Checking the clock.", "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": "Checking the clock.", "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": "Checking the clock.", "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:1:8": { "content": "Checking the clock.", "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:2:8": { "content": "Checking the clock.", "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:3:8": { "content": "Checking the clock.", "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:4:8": { "content": "Checking the clock.", "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:5:8": { "content": "Checking the clock.", "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null } } } }, { "fixture_id": "mistral/v11-no-call", "family": "mistral", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Mistral-Small-3.2-24B-Instruct-2506", "tokenizer": { "repo": "mistralai/Mistral-Small-3.2-24B-Instruct-2506", "revision": "95a6d26c4bfb886c58daf9d3f7332c857cb27b43", "class": "MistralCommonBackend", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "mistral", "hf_config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "model_type": "mistral3" }, "hf_config_error": null, "chat_template_sha256": "6caf38d90087d5c1c405ca0a28fce57478ad0312ab214e0fa9516028e07eaf4a", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 1091, 1047, 3174, 3074, 1093 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "mistral/v3-single-call", "family": "mistral", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Paris\", \"unit\": \"c\"}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Paris\", \"unit\": \"c\"}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Paris\", \"unit\": \"c\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Paris\", \"unit\": \"c\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"c\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Mistral-7B-Instruct-v0.3", "tokenizer": { "repo": "mistralai/Mistral-7B-Instruct-v0.3", "revision": "c170c708c41dac9275d15a8fff4eca08d52bab71", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "MistralForCausalLM" ], "model_type": "mistral" }, "hf_config_error": null, "chat_template_sha256": "e16746b40344d6c5b5265988e0328a0bf7277be86f1c335156eae07e29c82826", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 4 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null } } } }, { "fixture_id": "mistral/v3-parallel-calls", "family": "mistral", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'search'], got ['get_weather']" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'search'], got ['get_weather']" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected None, got ', {\"name\": \"search\", \"arguments\": {\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\"}}}'; tool_calls: expected ['get_weather', 'search'], got ['get_weather']" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected None, got ', {\"name\": \"search\", \"arguments\": {\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\"}}}'; tool_calls: expected ['get_weather', 'search'], got ['get_weather']" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected None, got ', {\"name\": \"search\", \"arguments\": {\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\"}}}'; tool_calls: expected ['get_weather', 'search'], got ['get_weather']" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected None, got ', {\"name\": \"search\", \"arguments\": {\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\"}}}'; tool_calls: expected ['get_weather', 'search'], got ['get_weather']" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected None, got ', {\"name\": \"search\", \"arguments\": {\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\"}}}'; tool_calls: expected ['get_weather', 'search'], got ['get_weather']" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected None, got ', {\"name\": \"search\", \"arguments\": {\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\"}}}'; tool_calls: expected ['get_weather', 'search'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'search'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'search'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream None, got ', {\"name\": \"search\", \"arguments\": {\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\"}}}'; tool_calls: nonstream ['get_weather', 'search'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream None, got ', {\"name\": \"search\", \"arguments\": {\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\"}}}'; tool_calls: nonstream ['get_weather', 'search'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream None, got ', {\"name\": \"search\", \"arguments\": {\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\"}}}'; tool_calls: nonstream ['get_weather', 'search'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream None, got ', {\"name\": \"search\", \"arguments\": {\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\"}}}'; tool_calls: nonstream ['get_weather', 'search'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream None, got ', {\"name\": \"search\", \"arguments\": {\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\"}}}'; tool_calls: nonstream ['get_weather', 'search'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream None, got ', {\"name\": \"search\", \"arguments\": {\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\"}}}'; tool_calls: nonstream ['get_weather', 'search'], got ['get_weather']" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content: one None, got ', {\"name\": \"search\", \"arguments\": {\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\"}}}'; tool_calls[0].arguments: one '', got '{\"city\": \"Zürich\"}' | rand:1:8 (fail): content: one None, got ', {\"name\": \"search\", \"arguments\": {\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\"}}}'; tool_calls[0].arguments: one '', got '{\"city\": \"Zürich\"}' | rand:2:8 (fail): content: one None, got ', {\"name\": \"search\", \"arguments\": {\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\"}}}'; tool_calls[0].arguments: one '', got '{\"city\": \"Zürich\"}' | rand:3:8 (fail): content: one None, got ', {\"name\": \"search\", \"arguments\": {\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\"}}}'; tool_calls[0].arguments: one '', got '{\"city\": \"Zürich\"}' | rand:4:8 (fail): content: one None, got ', {\"name\": \"search\", \"arguments\": {\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\"}}}'; tool_calls[0].arguments: one '', got '{\"city\": \"Zürich\"}' | rand:5:8 (fail): content: one None, got ', {\"name\": \"search\", \"arguments\": {\"query\": \"café \\\\\"best\\\\\"\", \"filters\": {\"tags\": [\"a\"}}}'; tool_calls[0].arguments: one '', got '{\"city\": \"Zürich\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "fail", "detail": "expected 2 calls ['get_weather', 'search'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "special", "status": "fail", "detail": "expected 2 calls ['get_weather', 'search'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "token", "status": "fail", "detail": "expected 2 calls ['get_weather', 'search'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "fail", "detail": "expected 2 calls ['get_weather', 'search'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "fail", "detail": "expected 2 calls ['get_weather', 'search'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "fail", "detail": "expected 2 calls ['get_weather', 'search'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "fail", "detail": "expected 2 calls ['get_weather', 'search'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "fail", "detail": "expected 2 calls ['get_weather', 'search'], got 1 ['get_weather']" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Mistral-7B-Instruct-v0.3", "tokenizer": { "repo": "mistralai/Mistral-7B-Instruct-v0.3", "revision": "c170c708c41dac9275d15a8fff4eca08d52bab71", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "MistralForCausalLM" ], "model_type": "mistral" }, "hf_config_error": null, "chat_template_sha256": "e16746b40344d6c5b5265988e0328a0bf7277be86f1c335156eae07e29c82826", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 4 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\"]}}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "token": { "content": ", {\"name\": \"search\", \"arguments\": {\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\"}}}", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\"}" } ], "exception": null }, "rand:1:8": { "content": ", {\"name\": \"search\", \"arguments\": {\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\"}}}", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\"}" } ], "exception": null }, "rand:2:8": { "content": ", {\"name\": \"search\", \"arguments\": {\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\"}}}", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\"}" } ], "exception": null }, "rand:3:8": { "content": ", {\"name\": \"search\", \"arguments\": {\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\"}}}", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\"}" } ], "exception": null }, "rand:4:8": { "content": ", {\"name\": \"search\", \"arguments\": {\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\"}}}", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\"}" } ], "exception": null }, "rand:5:8": { "content": ", {\"name\": \"search\", \"arguments\": {\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\"}}}", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\"}" } ], "exception": null } } } }, { "fixture_id": "mistral/v3-empty-arguments", "family": "mistral", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{}' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_time: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_time: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_time: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_time: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Mistral-7B-Instruct-v0.3", "tokenizer": { "repo": "mistralai/Mistral-7B-Instruct-v0.3", "revision": "c170c708c41dac9275d15a8fff4eca08d52bab71", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "MistralForCausalLM" ], "model_type": "mistral" }, "hf_config_error": null, "chat_template_sha256": "e16746b40344d6c5b5265988e0328a0bf7277be86f1c335156eae07e29c82826", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 4 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null } } } }, { "fixture_id": "mistral/v3-no-call", "family": "mistral", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "content (whitespace only): expected 'The capital of France is Paris.', got ' The capital of France is Paris.'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected 'The capital of France is Paris.', got ' The capital of France is Paris.'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected 'The capital of France is Paris.', got ' The capital of France is Paris.'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected 'The capital of France is Paris.', got ' The capital of France is Paris.'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'The capital of France is Paris.', got ' The capital of France is Paris.'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'The capital of France is Paris.', got ' The capital of France is Paris.'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'The capital of France is Paris.', got ' The capital of France is Paris.'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'The capital of France is Paris.', got ' The capital of France is Paris.'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'The capital of France is Paris.', got ' The capital of France is Paris.'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "mistral", "reasoning_parser": null, "tool_call_detector": "MistralDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "mistralai/Mistral-7B-Instruct-v0.3", "tokenizer": { "repo": "mistralai/Mistral-7B-Instruct-v0.3", "revision": "c170c708c41dac9275d15a8fff4eca08d52bab71", "class": "LlamaTokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "MistralForCausalLM" ], "model_type": "mistral" }, "hf_config_error": null, "chat_template_sha256": "e16746b40344d6c5b5265988e0328a0bf7277be86f1c335156eae07e29c82826", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 6, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 4 ] }, "stop": { "token": "", "id": 2, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": " The capital of France is Paris.", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": " The capital of France is Paris.", "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": " The capital of France is Paris.", "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": " The capital of France is Paris.", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:1:8": { "content": " The capital of France is Paris.", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:2:8": { "content": " The capital of France is Paris.", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:3:8": { "content": " The capital of France is Paris.", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:4:8": { "content": " The capital of France is Paris.", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:5:8": { "content": " The capital of France is Paris.", "reasoning_content": null, "tool_calls": [], "exception": null } } } }, { "fixture_id": "qwen3-hermes/single-call-thinking", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"\\nThe user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"\\nThe user wants the current weather in Paris. I'll call get_weather.\\n\"; tool_calls[0].arguments: expected '{\"city\": \"Paris\", \"unit\": \"c\"}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"\\nThe user wants the current weather in Paris. I'll call get_weather.\\n\"; tool_calls[0].arguments: expected '{\"city\": \"Paris\", \"unit\": \"c\"}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"\\nThe user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"\\nThe user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"\\nThe user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"\\nThe user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"\\nThe user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"\\nThe user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Paris\", \"unit\": \"c\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Paris\", \"unit\": \"c\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content (whitespace only): one None, got '\\n\\n'; tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:2:8 (fail): content (whitespace only): one None, got '\\n\\n'; tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"c\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "\nThe user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "\nThe user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": "\nThe user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "token": { "content": "\n\n", "reasoning_content": "\nThe user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "\nThe user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": "\nThe user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "\nThe user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "\nThe user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "\nThe user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/single-call-no-thinking", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Paris\", \"unit\": \"c\"}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Paris\", \"unit\": \"c\"}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Paris\", \"unit\": \"c\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Paris\", \"unit\": \"c\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"c\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 151667, 271, 151668, 271 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/no-call-thinking", "family": "qwen3-hermes", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "content (whitespace only): expected 'Hello! How can I help you today?', got '\\n\\nHello! How can I help you today?'; reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got '\\nThe user is just greeting me. No tool is needed.\\n'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected 'Hello! How can I help you today?', got '\\n\\nHello! How can I help you today?'; reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got '\\nThe user is just greeting me. No tool is needed.\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected 'Hello! How can I help you today?', got '\\n\\nHello! How can I help you today?'; reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got '\\nThe user is just greeting me. No tool is needed.\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected 'Hello! How can I help you today?', got '\\n\\nHello! How can I help you today?'; reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got '\\nThe user is just greeting me. No tool is needed.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Hello! How can I help you today?', got '\\n\\nHello! How can I help you today?'; reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got '\\nThe user is just greeting me. No tool is needed.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Hello! How can I help you today?', got '\\n\\nHello! How can I help you today?'; reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got '\\nThe user is just greeting me. No tool is needed.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Hello! How can I help you today?', got '\\n\\nHello! How can I help you today?'; reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got '\\nThe user is just greeting me. No tool is needed.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Hello! How can I help you today?', got '\\n\\nHello! How can I help you today?'; reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got '\\nThe user is just greeting me. No tool is needed.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Hello! How can I help you today?', got '\\n\\nHello! How can I help you today?'; reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got '\\nThe user is just greeting me. No tool is needed.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "\n\nHello! How can I help you today?", "reasoning_content": "\nThe user is just greeting me. No tool is needed.\n", "tool_calls": [], "exception": null }, "streams": { "one": { "content": "\n\nHello! How can I help you today?", "reasoning_content": "\nThe user is just greeting me. No tool is needed.\n", "tool_calls": [], "exception": null }, "special": { "content": "\n\nHello! How can I help you today?", "reasoning_content": "\nThe user is just greeting me. No tool is needed.\n", "tool_calls": [], "exception": null }, "token": { "content": "\n\nHello! How can I help you today?", "reasoning_content": "\nThe user is just greeting me. No tool is needed.\n", "tool_calls": [], "exception": null }, "rand:1:8": { "content": "\n\nHello! How can I help you today?", "reasoning_content": "\nThe user is just greeting me. No tool is needed.\n", "tool_calls": [], "exception": null }, "rand:2:8": { "content": "\n\nHello! How can I help you today?", "reasoning_content": "\nThe user is just greeting me. No tool is needed.\n", "tool_calls": [], "exception": null }, "rand:3:8": { "content": "\n\nHello! How can I help you today?", "reasoning_content": "\nThe user is just greeting me. No tool is needed.\n", "tool_calls": [], "exception": null }, "rand:4:8": { "content": "\n\nHello! How can I help you today?", "reasoning_content": "\nThe user is just greeting me. No tool is needed.\n", "tool_calls": [], "exception": null }, "rand:5:8": { "content": "\n\nHello! How can I help you today?", "reasoning_content": "\nThe user is just greeting me. No tool is needed.\n", "tool_calls": [], "exception": null } } } }, { "fixture_id": "qwen3-hermes/no-call-no-thinking", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 151667, 271, 151668, 271 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "qwen3-hermes/text-before-call-thinking", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I need live weather data for London.', got '\\nI need live weather data for London.\\n'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got None; reasoning_content (whitespace only): expected 'I need live weather data for London.', got '\\nI need live weather data for London.\\n'; tool_calls[0].arguments: expected '{\"city\": \"London\"}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got None; reasoning_content (whitespace only): expected 'I need live weather data for London.', got '\\nI need live weather data for London.\\n'; tool_calls[0].arguments: expected '{\"city\": \"London\"}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got '\\n\\nLet me check the current conditions in London.\\n'; reasoning_content (whitespace only): expected 'I need live weather data for London.', got '\\nI need live weather data for London.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got '\\n\\nLet me check the current conditions in London'; reasoning_content (whitespace only): expected 'I need live weather data for London.', got '\\nI need live weather data for London.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got '\\n\\nLet me check the current conditions in'; reasoning_content (whitespace only): expected 'I need live weather data for London.', got '\\nI need live weather data for London.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got '\\n\\nLet me check the current conditions in London.\\n'; reasoning_content (whitespace only): expected 'I need live weather data for London.', got '\\nI need live weather data for London.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got '\\n\\nLet me check the current conditions'; reasoning_content (whitespace only): expected 'I need live weather data for London.', got '\\nI need live weather data for London.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got '\\n\\nLet me check the current conditions in London'; reasoning_content (whitespace only): expected 'I need live weather data for London.', got '\\nI need live weather data for London.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got None; tool_calls[0].arguments: nonstream '{\"city\": \"London\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got None; tool_calls[0].arguments: nonstream '{\"city\": \"London\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Let me check the current conditions in London.', got '\\n\\nLet me check the current conditions in London.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got '\\n\\nLet me check the current conditions in London'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got '\\n\\nLet me check the current conditions in'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Let me check the current conditions in London.', got '\\n\\nLet me check the current conditions in London.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got '\\n\\nLet me check the current conditions'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got '\\n\\nLet me check the current conditions in London'" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content: one None, got '\\n\\nLet me check the current conditions in London.\\n'; tool_calls[0].arguments: one '', got '{\"city\": \"London\"}' | rand:1:8 (fail): content: one None, got '\\n\\nLet me check the current conditions in London'; tool_calls[0].arguments: one '', got '{\"city\": \"London\"}' | rand:2:8 (fail): content: one None, got '\\n\\nLet me check the current conditions in'; tool_calls[0].arguments: one '', got '{\"city\": \"London\"}' | rand:3:8 (fail): content: one None, got '\\n\\nLet me check the current conditions in London.\\n'; tool_calls[0].arguments: one '', got '{\"city\": \"London\"}' | rand:4:8 (fail): content: one None, got '\\n\\nLet me check the current conditions'; tool_calls[0].arguments: one '', got '{\"city\": \"London\"}' | rand:5:8 (fail): content: one None, got '\\n\\nLet me check the current conditions in London'; tool_calls[0].arguments: one '', got '{\"city\": \"London\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "Let me check the current conditions in London.", "reasoning_content": "\nI need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "\nI need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": "\nI need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "token": { "content": "\n\nLet me check the current conditions in London.\n", "reasoning_content": "\nI need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:1:8": { "content": "\n\nLet me check the current conditions in London", "reasoning_content": "\nI need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:2:8": { "content": "\n\nLet me check the current conditions in", "reasoning_content": "\nI need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:3:8": { "content": "\n\nLet me check the current conditions in London.\n", "reasoning_content": "\nI need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:4:8": { "content": "\n\nLet me check the current conditions", "reasoning_content": "\nI need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:5:8": { "content": "\n\nLet me check the current conditions in London", "reasoning_content": "\nI need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/text-before-call-no-thinking", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got None; tool_calls[0].arguments: expected '{\"city\": \"London\"}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got None; tool_calls[0].arguments: expected '{\"city\": \"London\"}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got 'Let me check the current'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got 'Let me check the current'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got 'Let me check the current'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got None; tool_calls[0].arguments: nonstream '{\"city\": \"London\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got None; tool_calls[0].arguments: nonstream '{\"city\": \"London\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got 'Let me check the current'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got 'Let me check the current'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got 'Let me check the current conditions in'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got 'Let me check the current'" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content: one None, got 'Let me check the current conditions in London.\\n'; tool_calls[0].arguments: one '', got '{\"city\": \"London\"}' | rand:1:8 (fail): content: one None, got 'Let me check the current'; tool_calls[0].arguments: one '', got '{\"city\": \"London\"}' | rand:2:8 (fail): content: one None, got 'Let me check the current'; tool_calls[0].arguments: one '', got '{\"city\": \"London\"}' | rand:3:8 (fail): content: one None, got 'Let me check the current conditions in'; tool_calls[0].arguments: one '', got '{\"city\": \"London\"}' | rand:4:8 (fail): content: one None, got 'Let me check the current conditions in London.\\n'; tool_calls[0].arguments: one '', got '{\"city\": \"London\"}' | rand:5:8 (fail): content: one None, got 'Let me check the current'; tool_calls[0].arguments: one '', got '{\"city\": \"London\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 151667, 271, 151668, 271 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "Let me check the current conditions in London.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "token": { "content": "Let me check the current conditions in London.\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:1:8": { "content": "Let me check the current", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:2:8": { "content": "Let me check the current", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:3:8": { "content": "Let me check the current conditions in", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:4:8": { "content": "Let me check the current conditions in London.\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:5:8": { "content": "Let me check the current", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/empty-arguments-no-thinking", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{}' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_time: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_time: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_time: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_time: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 151667, 271, 151668, 271 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/empty-arguments-thinking", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'get_time takes no arguments.', got '\\nget_time takes no arguments.\\n'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "reasoning_content (whitespace only): expected 'get_time takes no arguments.', got '\\nget_time takes no arguments.\\n'; tool_calls[0].arguments: expected '{}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "reasoning_content (whitespace only): expected 'get_time takes no arguments.', got '\\nget_time takes no arguments.\\n'; tool_calls[0].arguments: expected '{}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'get_time takes no arguments.', got '\\nget_time takes no arguments.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'get_time takes no arguments.', got '\\nget_time takes no arguments.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'get_time takes no arguments.', got '\\nget_time takes no arguments.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'get_time takes no arguments.', got '\\nget_time takes no arguments.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "reasoning_content (whitespace only): expected 'get_time takes no arguments.', got '\\nget_time takes no arguments.\\n'; tool_calls[0].arguments: expected '{}', got ''" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'get_time takes no arguments.', got '\\nget_time takes no arguments.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content (whitespace only): one None, got '\\n\\n'; tool_calls[0].arguments: one '', got '{}' | rand:1:8 (fail): content (whitespace only): one None, got '\\n\\n'; tool_calls[0].arguments: one '', got '{}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_time: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_time: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "fail", "detail": "[0] get_time: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_time: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_time: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[0] get_time: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "\nget_time takes no arguments.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "\nget_time takes no arguments.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": "\nget_time takes no arguments.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "" } ], "exception": null }, "token": { "content": "\n\n", "reasoning_content": "\nget_time takes no arguments.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:1:8": { "content": "\n\n", "reasoning_content": "\nget_time takes no arguments.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "\nget_time takes no arguments.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "\nget_time takes no arguments.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "\nget_time takes no arguments.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "\nget_time takes no arguments.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/bug-sglang-30480-truncated-mid-arguments", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Cut by max_tokens mid-arguments. The issue reports non-streaming leaking the raw markup into content while streaming drops it; a correct parser returns no call and does not leak markup." }, { "check": "expected_error", "strategy": "special", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Cut by max_tokens mid-arguments. The issue reports non-streaming leaking the raw markup into content while streaming drops it; a correct parser returns no call and does not leak markup." }, { "check": "expected_error", "strategy": "token", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Cut by max_tokens mid-arguments. The issue reports non-streaming leaking the raw markup into content while streaming drops it; a correct parser returns no call and does not leak markup." }, { "check": "expected_error", "strategy": "rand:1:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Cut by max_tokens mid-arguments. The issue reports non-streaming leaking the raw markup into content while streaming drops it; a correct parser returns no call and does not leak markup." }, { "check": "expected_error", "strategy": "rand:2:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Cut by max_tokens mid-arguments. The issue reports non-streaming leaking the raw markup into content while streaming drops it; a correct parser returns no call and does not leak markup." }, { "check": "expected_error", "strategy": "rand:3:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Cut by max_tokens mid-arguments. The issue reports non-streaming leaking the raw markup into content while streaming drops it; a correct parser returns no call and does not leak markup." }, { "check": "expected_error", "strategy": "rand:4:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Cut by max_tokens mid-arguments. The issue reports non-streaming leaking the raw markup into content while streaming drops it; a correct parser returns no call and does not leak markup." }, { "check": "expected_error", "strategy": "rand:5:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Cut by max_tokens mid-arguments. The issue reports non-streaming leaking the raw markup into content while streaming drops it; a correct parser returns no call and does not leak markup." }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream 'I will check.', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream 'I will check.', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content (whitespace only): nonstream 'I will check.', got 'I will check.\\n'; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream 'I will check.', got 'I will check'; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream 'I will check.', got 'I will check'; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content (whitespace only): nonstream 'I will check.', got 'I will check.\\n'; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content (whitespace only): nonstream 'I will check.', got 'I will check.\\n'; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream 'I will check.', got None; tool_calls: nonstream [], got ['get_weather']" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content: one None, got 'I will check.\\n'; tool_calls[0].arguments: one '', got '{\"city\": \"San Fr\"}' | rand:1:8 (fail): content: one None, got 'I will check'; tool_calls[0].arguments: one '', got '{\"city\": \"San Fr\"}' | rand:2:8 (fail): content: one None, got 'I will check'; tool_calls[0].arguments: one '', got '{\"city\": \"San Fr\"}' | rand:3:8 (fail): content: one None, got 'I will check.\\n'; tool_calls[0].arguments: one '', got '{\"city\": \"San Fr\"}' | rand:4:8 (fail): content: one None, got 'I will check.\\n'; tool_calls[0].arguments: one '', got '{\"city\": \"San Fr\"}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"San Fr\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 151667, 271, 151668, 271 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "I will check.", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "token": { "content": "I will check.\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"San Fr\"}" } ], "exception": null }, "rand:1:8": { "content": "I will check", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"San Fr\"}" } ], "exception": null }, "rand:2:8": { "content": "I will check", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"San Fr\"}" } ], "exception": null }, "rand:3:8": { "content": "I will check.\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"San Fr\"}" } ], "exception": null }, "rand:4:8": { "content": "I will check.\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"San Fr\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"San Fr\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/bug-sglang-30480-truncated-at-opener", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream 'I will check.\\n', got None" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream 'I will check.\\n', got None" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream 'I will check.\\n', got 'I will check.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream 'I will check.\\n', got 'I will check'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream 'I will check.\\n', got 'I will check'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream 'I will check.\\n', got 'I will check.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream 'I will check.\\n', got 'I will check.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream 'I will check.\\n', got None" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content: one None, got 'I will check.\\n' | rand:1:8 (fail): content: one None, got 'I will check' | rand:2:8 (fail): content: one None, got 'I will check' | rand:3:8 (fail): content: one None, got 'I will check.\\n' | rand:4:8 (fail): content: one None, got 'I will check.\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 151667, 271, 151668, 271 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "I will check.\n", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": "I will check.\n", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:1:8": { "content": "I will check", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:2:8": { "content": "I will check", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:3:8": { "content": "I will check.\n", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:4:8": { "content": "I will check.\n", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null } } } }, { "fixture_id": "qwen3-hermes/nested-json-deep", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"an..., got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"an..., got ''" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"an..., got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"an..., got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"an... | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"an... | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"an... | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"an... | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"an... | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"an..." }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] create_event: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] create_event: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] create_event: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] create_event: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 151667, 271, 151668, 271 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"ana@example.com\", \"optional\": false}, {\"name\": \"Bo\", \"email\": \"bo@example.com\", \"optional\": true, \"roles\": [\"notes\", \"timekeeper\"]}], \"reminders\": [[10, \"email\"], [1, \"popup\"]], \"location\": null}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"ana@example.com\", \"optional\": false}, {\"name\": \"Bo\", \"email\": \"bo@example.com\", \"optional\": true, \"roles\": [\"notes\", \"timekeeper\"]}], \"reminders\": [[10, \"email\"], [1, \"popup\"]], \"location\": null}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"ana@example.com\", \"optional\": false}, {\"name\": \"Bo\", \"email\": \"bo@example.com\", \"optional\": true, \"roles\": [\"notes\", \"timekeeper\"]}], \"reminders\": [[10, \"email\"], [1, \"popup\"]], \"location\": null}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"ana@example.com\", \"optional\": false}, {\"name\": \"Bo\", \"email\": \"bo@example.com\", \"optional\": true, \"roles\": [\"notes\", \"timekeeper\"]}], \"reminders\": [[10, \"email\"], [1, \"popup\"]], \"location\": null}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"ana@example.com\", \"optional\": false}, {\"name\": \"Bo\", \"email\": \"bo@example.com\", \"optional\": true, \"roles\": [\"notes\", \"timekeeper\"]}], \"reminders\": [[10, \"email\"], [1, \"popup\"]], \"location\": null}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"ana@example.com\", \"optional\": false}, {\"name\": \"Bo\", \"email\": \"bo@example.com\", \"optional\": true, \"roles\": [\"notes\", \"timekeeper\"]}], \"reminders\": [[10, \"email\"], [1, \"popup\"]], \"location\": null}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "create_event", "arguments_raw": "{\"title\": \"Design review\", \"when\": {\"start\": \"2026-10-01T09:00:00Z\", \"end\": \"2026-10-01T10:00:00Z\", \"tz\": \"UTC\"}, \"attendees\": [{\"name\": \"Ana\", \"email\": \"ana@example.com\", \"optional\": false}, {\"name\": \"Bo\", \"email\": \"bo@example.com\", \"optional\": true, \"roles\": [\"notes\", \"timekeeper\"]}], \"reminders\": [[10, \"email\"], [1, \"popup\"]], \"location\": null}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/unicode-emoji-thinking", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Translate the mixed-script greeting (日本語, العربية, emoji 🌸).', got '\\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\\n'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "reasoning_content (whitespace only): expected 'Translate the mixed-script greeting (日本語, العربية, emoji 🌸).', got '\\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\\n'; tool_calls[0].arguments: expected '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "reasoning_content (whitespace only): expected 'Translate the mixed-script greeting (日本語, العربية, emoji 🌸).', got '\\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\\n'; tool_calls[0].arguments: expected '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Translate the mixed-script greeting (日本語, العربية, emoji 🌸).', got '\\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Translate the mixed-script greeting (日本語, العربية, emoji 🌸).', got '\\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Translate the mixed-script greeting (日本語, العربية, emoji 🌸).', got '\\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Translate the mixed-script greeting (日本語, العربية, emoji 🌸).', got '\\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Translate the mixed-script greeting (日本語, العربية, emoji 🌸).', got '\\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Translate the mixed-script greeting (日本語, العربية, emoji 🌸).', got '\\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content (whitespace only): one None, got '\\n\\n'; tool_calls[0].arguments: one '', got '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}' | rand:1:8 (fail): content (whitespace only): one None, got '\\n\\n'; tool_calls[0].arguments: one '', got '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}' | rand:5:8 (fail): content (whitespace only): one None, got '\\n\\n'; tool_calls[0].arguments: one '', got '{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩\\u200d💻\", \"target\": \"de\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] translate: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] translate: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] translate: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] translate: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\n", "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\n", "tool_calls": [ { "name": "translate", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": "\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\n", "tool_calls": [ { "name": "translate", "arguments_raw": "" } ], "exception": null }, "token": { "content": "\n\n", "reasoning_content": "\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\n", "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null }, "rand:1:8": { "content": "\n\n", "reasoning_content": "\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\n", "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\n", "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\n", "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\n", "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null }, "rand:5:8": { "content": "\n\n", "reasoning_content": "\nTranslate the mixed-script greeting (日本語, العربية, emoji 🌸).\n", "tool_calls": [ { "name": "translate", "arguments_raw": "{\"text\": \"こんにちは 🌸 مرحبا — naïve café 👩‍💻\", \"target\": \"de\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/marker-in-arguments", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "reasoning_content: expected None, got '\\\\\" blocks.\"}}\\n'; tool_calls[0].arguments: expected '{\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}', got '{\"text\": \"Use ... tags, or \\\\\"\"}'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "reasoning_content: expected None, got ' tags, or \\\\\"\\\\\" blocks.\"}}\\n'; tool_calls[0].arguments: expected '{\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}', got '{\"text\": \"Use ...\"}'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "reasoning_content: expected None, got ' \\\\\"\\\\\" blocks.\"}}\\n'; tool_calls[0].arguments: expected '{\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}', got '{\"text\": \"Use ... tags, or\"}'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "reasoning_content: expected None, got ' \\\\\"\\\\\" blocks.\"}}\\n'; tool_calls[0].arguments: expected '{\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}', got '{\"text\": \"Use ... tags, or\"}'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "reasoning_content: expected None, got ' ... tags, or \\\\\"\\\\\" blocks.\"}}\\n'; tool_calls[0].arguments: expected '{\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}', got '{\"text\": \"Use \"}'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "reasoning_content: expected None, got '\\\\\" blocks.\"}}\\n'; tool_calls[0].arguments: expected '{\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}', got '{\"text\": \"Use ... tags, or \\\\\"\"}'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "reasoning_content: nonstream None, got '\\\\\" blocks.\"}}\\n'; tool_calls[0].arguments: nonstream '{\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}', got '{\"text\": \"Use ... tags, or \\\\\"\"}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "reasoning_content: nonstream None, got ' tags, or \\\\\"\\\\\" blocks.\"}}\\n'; tool_calls[0].arguments: nonstream '{\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}', got '{\"text\": \"Use ...\"}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "reasoning_content: nonstream None, got ' \\\\\"\\\\\" blocks.\"}}\\n'; tool_calls[0].arguments: nonstream '{\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}', got '{\"text\": \"Use ... tags, or\"}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "reasoning_content: nonstream None, got ' \\\\\"\\\\\" blocks.\"}}\\n'; tool_calls[0].arguments: nonstream '{\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}', got '{\"text\": \"Use ... tags, or\"}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "reasoning_content: nonstream None, got ' ... tags, or \\\\\"\\\\\" blocks.\"}}\\n'; tool_calls[0].arguments: nonstream '{\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}', got '{\"text\": \"Use \"}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "reasoning_content: nonstream None, got '\\\\\" blocks.\"}}\\n'; tool_calls[0].arguments: nonstream '{\"text\": \"Use ... tags, or \\\\\"\\\\\" blocks.\"}', got '{\"text\": \"Use ... tags, or \\\\\"\"}'" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): reasoning_content: one None, got '\\\\\" blocks.\"}}\\n'; tool_calls[0].arguments: one '', got '{\"text\": \"Use ... tags, or \\\\\"\"}' | rand:1:8 (fail): reasoning_content: one None, got ' tags, or \\\\\"\\\\\" blocks.\"}}\\n'; tool_calls[0].arguments: one '', got '{\"text\": \"Use ...\"}' | rand:2:8 (fail): reasoning_content: one None, got ' \\\\\"\\\\\" blocks.\"}}\\n'; tool_calls[0].arguments: one '', got '{\"text\": \"Use ... tags, or\"}' | rand:3:8 (fail): reasoning_content: one None, got ' \\\\\"\\\\\" blocks.\"}}\\n'; tool_calls[0].arguments: one '', got '{\"text\": \"Use ... tags, or\"}' | rand:4:8 (fail): reasoning_content: one None, got ' ... tags, or \\\\\"\\\\\" blocks.\"}}\\n'; tool_calls[0].arguments: one '', got '{\"text\": \"Use \"}' | rand:5:8 (fail): reasoning_content: one None, got '\\\\\" blocks.\"}}\\n'; tool_calls[0].arguments: one '', got '{\"text\": \"Use ... tags, or \\\\\"\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "fail", "detail": "reasoning_content contains ''" }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "fail", "detail": "reasoning_content contains ''" }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "fail", "detail": "reasoning_content contains ''" }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "fail", "detail": "reasoning_content contains ''" }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "fail", "detail": "reasoning_content contains ''" }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "fail", "detail": "reasoning_content contains ''" }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] echo: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] echo: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] echo: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] echo: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 151667, 271, 151668, 271 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Use ... tags, or \\\"\\\" blocks.\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": "\\\" blocks.\"}}\n", "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Use ... tags, or \\\"\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": " tags, or \\\"\\\" blocks.\"}}\n", "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Use ...\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": " \\\"\\\" blocks.\"}}\n", "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Use ... tags, or\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": " \\\"\\\" blocks.\"}}\n", "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Use ... tags, or\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": " ... tags, or \\\"\\\" blocks.\"}}\n", "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Use \"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "\\\" blocks.\"}}\n", "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"Use ... tags, or \\\"\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/string-escapes-code", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"C:\\\\\\\\Users\\\\\\\\me\\\\\\\\hello.py\", \"content\": \"def main():\\\\n\\\\tprint(\\\\\"hi \\\\\\\\u00e9\\\\\")\\\\n\\\\treturn {\\\\\"ok\\\\\": True}\\\\n\"}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"C:\\\\\\\\Users\\\\\\\\me\\\\\\\\hello.py\", \"content\": \"def main():\\\\n\\\\tprint(\\\\\"hi \\\\\\\\u00e9\\\\\")\\\\n\\\\treturn {\\\\\"ok\\\\\": True}\\\\n\"}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"path\": \"C:\\\\\\\\Users\\\\\\\\me\\\\\\\\hello.py\", \"content\": \"def main():\\\\n\\\\tprint(\\\\\"hi \\\\\\\\u00e9\\\\\")\\\\n\\\\treturn {\\\\\"ok\\\\\": True}\\\\n\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"path\": \"C:\\\\\\\\Users\\\\\\\\me\\\\\\\\hello.py\", \"content\": \"def main():\\\\n\\\\tprint(\\\\\"hi \\\\\\\\u00e9\\\\\")\\\\n\\\\treturn {\\\\\"ok\\\\\": True}\\\\n\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{\"path\": \"C:\\\\\\\\Users\\\\\\\\me\\\\\\\\hello.py\", \"content\": \"def main():\\\\n\\\\tprint(\\\\\"hi \\\\\\\\u00e9\\\\\")\\\\n\\\\treturn {\\\\\"ok\\\\\": True}\\\\n\"}' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"path\": \"C:\\\\\\\\Users\\\\\\\\me\\\\\\\\hello.py\", \"content\": \"def main():\\\\n\\\\tprint(\\\\\"hi \\\\\\\\u00e9\\\\\")\\\\n\\\\treturn {\\\\\"ok\\\\\": True}\\\\n\"}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{\"path\": \"C:\\\\\\\\Users\\\\\\\\me\\\\\\\\hello.py\", \"content\": \"def main():\\\\n\\\\tprint(\\\\\"hi \\\\\\\\u00e9\\\\\")\\\\n\\\\treturn {\\\\\"ok\\\\\": True}\\\\n\"}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"path\": \"C:\\\\\\\\Users\\\\\\\\me\\\\\\\\hello.py\", \"content\": \"def main():\\\\n\\\\tprint(\\\\\"hi \\\\\\\\u00e9\\\\\")\\\\n\\\\treturn {\\\\\"ok\\\\\": True}\\\\n\"}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"path\": \"C:\\\\\\\\Users\\\\\\\\me\\\\\\\\hello.py\", \"content\": \"def main():\\\\n\\\\tprint(\\\\\"hi \\\\\\\\u00e9\\\\\")\\\\n\\\\treturn {\\\\\"ok\\\\\": True}\\\\n\"}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"path\": \"C:\\\\\\\\Users\\\\\\\\me\\\\\\\\hello.py\", \"content\": \"def main():\\\\n\\\\tprint(\\\\\"hi \\\\\\\\u00e9\\\\\")\\\\n\\\\treturn {\\\\\"ok\\\\\": True}\\\\n\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 151667, 271, 151668, 271 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"C:\\\\Users\\\\me\\\\hello.py\", \"content\": \"def main():\\n\\tprint(\\\"hi \\\\u00e9\\\")\\n\\treturn {\\\"ok\\\": True}\\n\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"C:\\\\Users\\\\me\\\\hello.py\", \"content\": \"def main():\\n\\tprint(\\\"hi \\\\u00e9\\\")\\n\\treturn {\\\"ok\\\": True}\\n\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"C:\\\\Users\\\\me\\\\hello.py\", \"content\": \"def main():\\n\\tprint(\\\"hi \\\\u00e9\\\")\\n\\treturn {\\\"ok\\\": True}\\n\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"C:\\\\Users\\\\me\\\\hello.py\", \"content\": \"def main():\\n\\tprint(\\\"hi \\\\u00e9\\\")\\n\\treturn {\\\"ok\\\": True}\\n\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"C:\\\\Users\\\\me\\\\hello.py\", \"content\": \"def main():\\n\\tprint(\\\"hi \\\\u00e9\\\")\\n\\treturn {\\\"ok\\\": True}\\n\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"C:\\\\Users\\\\me\\\\hello.py\", \"content\": \"def main():\\n\\tprint(\\\"hi \\\\u00e9\\\")\\n\\treturn {\\\"ok\\\": True}\\n\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"C:\\\\Users\\\\me\\\\hello.py\", \"content\": \"def main():\\n\\tprint(\\\"hi \\\\u00e9\\\")\\n\\treturn {\\\"ok\\\": True}\\n\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/json-string-argument", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"text\": \"{\\\\\"name\\\\\": \\\\\"x\\\\\", \\\\\"arguments\\\\\": {\\\\\"a\\\\\": 1}}\"}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"text\": \"{\\\\\"name\\\\\": \\\\\"x\\\\\", \\\\\"arguments\\\\\": {\\\\\"a\\\\\": 1}}\"}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"{\\\\\"name\\\\\": \\\\\"x\\\\\", \\\\\"arguments\\\\\": {\\\\\"a\\\\\": 1}}\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"text\": \"{\\\\\"name\\\\\": \\\\\"x\\\\\", \\\\\"arguments\\\\\": {\\\\\"a\\\\\": 1}}\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{\"text\": \"{\\\\\"name\\\\\": \\\\\"x\\\\\", \\\\\"arguments\\\\\": {\\\\\"a\\\\\": 1}}\"}' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"text\": \"{\\\\\"name\\\\\": \\\\\"x\\\\\", \\\\\"arguments\\\\\": {\\\\\"a\\\\\": 1}}\"}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{\"text\": \"{\\\\\"name\\\\\": \\\\\"x\\\\\", \\\\\"arguments\\\\\": {\\\\\"a\\\\\": 1}}\"}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"text\": \"{\\\\\"name\\\\\": \\\\\"x\\\\\", \\\\\"arguments\\\\\": {\\\\\"a\\\\\": 1}}\"}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"text\": \"{\\\\\"name\\\\\": \\\\\"x\\\\\", \\\\\"arguments\\\\\": {\\\\\"a\\\\\": 1}}\"}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"text\": \"{\\\\\"name\\\\\": \\\\\"x\\\\\", \\\\\"arguments\\\\\": {\\\\\"a\\\\\": 1}}\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] echo: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] echo: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] echo: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] echo: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 151667, 271, 151668, 271 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"{\\\"name\\\": \\\"x\\\", \\\"arguments\\\": {\\\"a\\\": 1}}\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"{\\\"name\\\": \\\"x\\\", \\\"arguments\\\": {\\\"a\\\": 1}}\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"{\\\"name\\\": \\\"x\\\", \\\"arguments\\\": {\\\"a\\\": 1}}\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"{\\\"name\\\": \\\"x\\\", \\\"arguments\\\": {\\\"a\\\": 1}}\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"{\\\"name\\\": \\\"x\\\", \\\"arguments\\\": {\\\"a\\\": 1}}\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"{\\\"name\\\": \\\"x\\\", \\\"arguments\\\": {\\\"a\\\": 1}}\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "echo", "arguments_raw": "{\"text\": \"{\\\"name\\\": \\\"x\\\", \\\"arguments\\\": {\\\"a\\\": 1}}\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/numeric-arguments", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21]}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21]}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21]}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21]}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21]}' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21]}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21]}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21]}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21]}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21]}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] calculate: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] calculate: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] calculate: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] calculate: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 151667, 271, 151668, 271 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21]}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21]}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21]}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21]}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21]}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21]}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "calculate", "arguments_raw": "{\"x\": -3, \"y\": 2.5, \"tolerance\": 1e-05, \"exact\": true, \"limit\": null, \"values\": [0, -0.5, 1e+21]}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/long-arguments", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"notes/sample.txt\", \"content\": \"001: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\\\n002: The quick brown fox jumps over the lazy d..., got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"notes/sample.txt\", \"content\": \"001: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\\\n002: The quick brown fox jumps over the lazy d..., got ''" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"path\": \"notes/sample.txt\", \"content\": \"001: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\\\n002: The quick brown fox jumps over the lazy d..., got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"path\": \"notes/sample.txt\", \"content\": \"001: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\\\n002: The quick brown fox jumps over the lazy d..., got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{\"path\": \"notes/sample.txt\", \"content\": \"001: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\\\n002: The quick brown fox jumps over the lazy d... | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"path\": \"notes/sample.txt\", \"content\": \"001: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\\\n002: The quick brown fox jumps over the lazy d... | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{\"path\": \"notes/sample.txt\", \"content\": \"001: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\\\n002: The quick brown fox jumps over the lazy d... | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"path\": \"notes/sample.txt\", \"content\": \"001: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\\\n002: The quick brown fox jumps over the lazy d... | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"path\": \"notes/sample.txt\", \"content\": \"001: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\\\n002: The quick brown fox jumps over the lazy d... | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"path\": \"notes/sample.txt\", \"content\": \"001: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\\\n002: The quick brown fox jumps over the lazy d..." }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 151667, 271, 151668, 271 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"notes/sample.txt\", \"content\": \"001: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n002: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n003: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n004: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n005: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n006: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n007: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n008: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n009: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n010: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n011: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n012: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n013: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n014: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n015: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n016: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n017: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n018: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n019: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n020: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n021: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n022: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n023: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n024: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n025: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n026: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n027: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n028: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n029: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n030: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n031: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n032: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n033: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n034: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n035: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n036: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n037: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n038: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n039: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n040: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n041: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n042: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n043: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n044: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n045: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n046: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n047: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n048: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n049: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n050: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n051: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n052: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n053: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n054: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n055: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n056: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n057: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n058: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n059: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n060: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"notes/sample.txt\", \"content\": \"001: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n002: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n003: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n004: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n005: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n006: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n007: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n008: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n009: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n010: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n011: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n012: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n013: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n014: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n015: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n016: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n017: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n018: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n019: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n020: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n021: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n022: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n023: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n024: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n025: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n026: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n027: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n028: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n029: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n030: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n031: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n032: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n033: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n034: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n035: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n036: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n037: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n038: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n039: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n040: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n041: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n042: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n043: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n044: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n045: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n046: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n047: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n048: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n049: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n050: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n051: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n052: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n053: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n054: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n055: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n056: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n057: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n058: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n059: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n060: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"notes/sample.txt\", \"content\": \"001: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n002: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n003: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n004: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n005: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n006: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n007: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n008: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n009: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n010: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n011: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n012: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n013: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n014: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n015: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n016: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n017: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n018: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n019: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n020: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n021: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n022: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n023: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n024: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n025: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n026: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n027: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n028: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n029: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n030: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n031: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n032: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n033: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n034: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n035: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n036: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n037: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n038: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n039: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n040: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n041: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n042: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n043: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n044: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n045: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n046: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n047: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n048: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n049: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n050: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n051: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n052: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n053: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n054: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n055: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n056: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n057: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n058: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n059: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n060: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"notes/sample.txt\", \"content\": \"001: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n002: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n003: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n004: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n005: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n006: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n007: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n008: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n009: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n010: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n011: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n012: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n013: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n014: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n015: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n016: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n017: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n018: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n019: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n020: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n021: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n022: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n023: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n024: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n025: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n026: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n027: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n028: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n029: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n030: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n031: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n032: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n033: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n034: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n035: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n036: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n037: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n038: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n039: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n040: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n041: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n042: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n043: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n044: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n045: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n046: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n047: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n048: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n049: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n050: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n051: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n052: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n053: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n054: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n055: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n056: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n057: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n058: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n059: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n060: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"notes/sample.txt\", \"content\": \"001: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n002: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n003: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n004: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n005: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n006: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n007: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n008: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n009: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n010: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n011: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n012: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n013: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n014: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n015: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n016: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n017: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n018: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n019: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n020: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n021: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n022: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n023: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n024: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n025: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n026: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n027: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n028: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n029: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n030: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n031: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n032: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n033: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n034: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n035: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n036: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n037: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n038: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n039: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n040: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n041: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n042: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n043: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n044: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n045: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n046: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n047: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n048: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n049: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n050: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n051: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n052: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n053: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n054: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n055: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n056: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n057: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n058: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n059: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n060: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"notes/sample.txt\", \"content\": \"001: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n002: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n003: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n004: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n005: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n006: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n007: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n008: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n009: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n010: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n011: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n012: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n013: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n014: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n015: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n016: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n017: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n018: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n019: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n020: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n021: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n022: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n023: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n024: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n025: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n026: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n027: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n028: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n029: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n030: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n031: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n032: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n033: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n034: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n035: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n036: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n037: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n038: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n039: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n040: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n041: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n042: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n043: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n044: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n045: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n046: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n047: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n048: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n049: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n050: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n051: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n052: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n053: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n054: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n055: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n056: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n057: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n058: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n059: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n060: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"notes/sample.txt\", \"content\": \"001: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n002: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n003: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n004: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n005: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n006: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n007: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n008: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n009: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n010: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n011: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n012: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n013: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n014: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n015: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n016: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n017: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n018: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n019: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n020: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n021: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n022: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n023: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n024: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n025: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n026: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n027: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n028: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n029: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n030: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n031: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n032: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n033: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n034: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n035: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n036: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n037: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n038: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n039: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n040: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n041: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n042: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n043: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n044: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n045: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n046: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n047: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n048: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n049: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n050: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n051: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n052: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n053: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n054: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n055: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n056: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n057: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n058: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n059: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n060: The quick brown fox jumps over the lazy dog; café, naïve, 東京.\\n\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/vllm-plain-text-forwarded", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 151667, 271, 151668, 271 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "qwen3-hermes/vllm-bug-19056-boolean-argument", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"trigger\": true}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"trigger\": true}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"trigger\": true}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"trigger\": true}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{\"trigger\": true}' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"trigger\": true}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{\"trigger\": true}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"trigger\": true}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"trigger\": true}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"trigger\": true}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] final_answer: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] final_answer: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] final_answer: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] final_answer: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 151667, 271, 151668, 271 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "final_answer", "arguments_raw": "{\"trigger\": true}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "final_answer", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "final_answer", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "final_answer", "arguments_raw": "{\"trigger\": true}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "final_answer", "arguments_raw": "{\"trigger\": true}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "final_answer", "arguments_raw": "{\"trigger\": true}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "final_answer", "arguments_raw": "{\"trigger\": true}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "final_answer", "arguments_raw": "{\"trigger\": true}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "final_answer", "arguments_raw": "{\"trigger\": true}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/vllm-no-newlines-no-spaces", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}'; tool_calls: expected ['get_current_temperature'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected None, got '{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}'; tool_calls: expected ['get_current_temperature'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected None, got '{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}'; tool_calls: expected ['get_current_temperature'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected None, got '{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}'; tool_calls: expected ['get_current_temperature'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected None, got '{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}'; tool_calls: expected ['get_current_temperature'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected None, got '{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}'; tool_calls: expected ['get_current_temperature'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected None, got '{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}'; tool_calls: expected ['get_current_temperature'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected None, got '{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}'; tool_calls: expected ['get_current_temperature'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected None, got '{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}'; tool_calls: expected ['get_current_temperature'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}', got '{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}', got '{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}', got '{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}', got '{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}', got '{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}', got '{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}', got '{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}', got '{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}'" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "one", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "special", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "token", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "fail", "detail": "content contains ''" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 151667, 271, 151668, 271 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": "{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": "{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": "{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:1:8": { "content": "{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:2:8": { "content": "{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:3:8": { "content": "{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:4:8": { "content": "{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:5:8": { "content": "{\"name\": \"get_current_temperature\",\"arguments\": {\"location\":\"San Francisco, California, United States\", \"unit\": \"celsius\"}}", "reasoning_content": null, "tool_calls": [], "exception": null } } } }, { "fixture_id": "qwen3-hermes/vllm-text-then-call-no-separator", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected 'Sure, let me check the weather.', got 'Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected 'Sure, let me check the weather.', got 'Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected 'Sure, let me check the weather.', got 'Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected 'Sure, let me check the weather.', got 'Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected 'Sure, let me check the weather.', got 'Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected 'Sure, let me check the weather.', got 'Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected 'Sure, let me check the weather.', got 'Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected 'Sure, let me check the weather.', got 'Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected 'Sure, let me check the weather.', got 'Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}'; tool_calls: expected ['get_weather'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream 'Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}', got 'Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream 'Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}', got 'Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream 'Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}', got 'Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream 'Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}', got 'Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream 'Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}', got 'Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream 'Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}', got 'Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream 'Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}', got 'Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream 'Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}', got 'Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}'" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "one", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "special", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "token", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "fail", "detail": "content contains ''" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 151667, 271, 151668, 271 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": "Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": "Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": "Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:1:8": { "content": "Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:2:8": { "content": "Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:3:8": { "content": "Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:4:8": { "content": "Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:5:8": { "content": "Sure, let me check the weather.{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}", "reasoning_content": null, "tool_calls": [], "exception": null } } } }, { "fixture_id": "qwen3-hermes/vllm-two-calls-no-separator", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}'; tool_calls: expected ['search', 'search'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected None, got '{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}'; tool_calls: expected ['search', 'search'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected None, got '{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}'; tool_calls: expected ['search', 'search'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected None, got '{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}'; tool_calls: expected ['search', 'search'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected None, got '{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}'; tool_calls: expected ['search', 'search'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected None, got '{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}'; tool_calls: expected ['search', 'search'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected None, got '{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}'; tool_calls: expected ['search', 'search'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected None, got '{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}'; tool_calls: expected ['search', 'search'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected None, got '{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}'; tool_calls: expected ['search', 'search'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}', got '{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}', got '{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}', got '{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}', got '{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}', got '{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}', got '{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}', got '{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}', got '{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}'" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "one", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "special", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "token", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "fail", "detail": "content contains ''" }, { "check": "parallel_order", "strategy": "nonstream", "status": "fail", "detail": "expected 2 calls ['search', 'search'], got 0 []" }, { "check": "parallel_order", "strategy": "one", "status": "fail", "detail": "expected 2 calls ['search', 'search'], got 0 []" }, { "check": "parallel_order", "strategy": "special", "status": "fail", "detail": "expected 2 calls ['search', 'search'], got 0 []" }, { "check": "parallel_order", "strategy": "token", "status": "fail", "detail": "expected 2 calls ['search', 'search'], got 0 []" }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "fail", "detail": "expected 2 calls ['search', 'search'], got 0 []" }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "fail", "detail": "expected 2 calls ['search', 'search'], got 0 []" }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "fail", "detail": "expected 2 calls ['search', 'search'], got 0 []" }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "fail", "detail": "expected 2 calls ['search', 'search'], got 0 []" }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "fail", "detail": "expected 2 calls ['search', 'search'], got 0 []" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 151667, 271, 151668, 271 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": "{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": "{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": "{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:1:8": { "content": "{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:2:8": { "content": "{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:3:8": { "content": "{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:4:8": { "content": "{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:5:8": { "content": "{\"name\": \"search\", \"arguments\": {\"q\": \"cats\"}}{\"name\": \"search\", \"arguments\": {\"q\": \"dogs\"}}", "reasoning_content": null, "tool_calls": [], "exception": null } } } }, { "fixture_id": "qwen3-hermes/vllm-content-and-call-single-chunk", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected 'Hi!', got 'Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}'; tool_calls: expected ['f'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected 'Hi!', got 'Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}'; tool_calls: expected ['f'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected 'Hi!', got 'Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}'; tool_calls: expected ['f'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected 'Hi!', got 'Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}'; tool_calls: expected ['f'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected 'Hi!', got 'Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}'; tool_calls: expected ['f'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected 'Hi!', got 'Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}'; tool_calls: expected ['f'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected 'Hi!', got 'Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}'; tool_calls: expected ['f'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected 'Hi!', got 'Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}'; tool_calls: expected ['f'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected 'Hi!', got 'Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}'; tool_calls: expected ['f'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream 'Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}', got 'Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream 'Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}', got 'Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream 'Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}', got 'Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream 'Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}', got 'Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream 'Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}', got 'Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream 'Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}', got 'Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream 'Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}', got 'Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream 'Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}', got 'Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}'" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "one", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "special", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "token", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "fail", "detail": "content contains ''" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 151667, 271, 151668, 271 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": "Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": "Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": "Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:1:8": { "content": "Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:2:8": { "content": "Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:3:8": { "content": "Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:4:8": { "content": "Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:5:8": { "content": "Hi!{\"name\": \"f\", \"arguments\": {\"x\": 1}}", "reasoning_content": null, "tool_calls": [], "exception": null } } } }, { "fixture_id": "qwen3-hermes/sglang-text-before-call-with-space", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected 'I will check the weather for you.', got 'I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected 'I will check the weather for you.', got 'I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected 'I will check the weather for you.', got 'I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected 'I will check the weather for you.', got 'I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected 'I will check the weather for you.', got 'I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected 'I will check the weather for you.', got 'I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected 'I will check the weather for you.', got 'I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected 'I will check the weather for you.', got 'I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'; tool_calls: expected ['get_weather'], got []" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected 'I will check the weather for you.', got 'I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'; tool_calls: expected ['get_weather'], got []" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream 'I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}', got 'I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream 'I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}', got 'I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream 'I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}', got 'I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream 'I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}', got 'I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream 'I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}', got 'I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream 'I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}', got 'I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream 'I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}', got 'I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream 'I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}', got 'I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}'" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "one", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "special", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "token", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "fail", "detail": "content contains ''" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 151667, 271, 151668, 271 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": "I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": "I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": "I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:1:8": { "content": "I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:2:8": { "content": "I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:3:8": { "content": "I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:4:8": { "content": "I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:5:8": { "content": "I will check the weather for you. {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}", "reasoning_content": null, "tool_calls": [], "exception": null } } } }, { "fixture_id": "qwen3-hermes/sglang-qwen25-four-parallel-calls", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_current_weather', 'get_current_weather', 'get_current_weather', 'get_current_weather'], got ['get_current_weather']" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['get_current_weather', 'get_current_weather', 'get_current_weather', 'get_current_weather'], got ['get_current_weather']" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream ['get_current_weather', 'get_current_weather', 'get_current_weather', 'get_current_weather'], got ['get_current_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls: nonstream ['get_current_weather', 'get_current_weather', 'get_current_weather', 'get_current_weather'], got ['get_current_weather']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls: one ['get_current_weather'], got ['get_current_weather', 'get_current_weather', 'get_current_weather', 'get_current_weather'] | rand:1:8 (fail): tool_calls: one ['get_current_weather'], got ['get_current_weather', 'get_current_weather', 'get_current_weather', 'get_current_weather'] | rand:2:8 (fail): tool_calls: one ['get_current_weather'], got ['get_current_weather', 'get_current_weather', 'get_current_weather', 'get_current_weather'] | rand:3:8 (fail): tool_calls: one ['get_current_weather'], got ['get_current_weather', 'get_current_weather', 'get_current_weather', 'get_current_weather'] | rand:4:8 (fail): tool_calls: one ['get_current_weather'], got ['get_current_weather', 'get_current_weather', 'get_current_weather', 'get_current_weather'] | rand:5:8 (fail): tool_calls: one ['get_current_weather'], got ['get_current_weather', 'get_current_weather', 'get_current_weather', 'get_current_weather']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_current_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_current_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_current_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_current_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "fail", "detail": "expected 4 calls ['get_current_weather', 'get_current_weather', 'get_current_weather', 'get_current_weather'], got 1 ['get_current_weather']" }, { "check": "parallel_order", "strategy": "special", "status": "fail", "detail": "expected 4 calls ['get_current_weather', 'get_current_weather', 'get_current_weather', 'get_current_weather'], got 1 ['get_current_weather']" }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 151667, 271, 151668, 271 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"NYC\", \"state\": \"NY\", \"unit\": \"fahrenheit\"}" }, { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Baltimore\", \"state\": \"MD\", \"unit\": \"fahrenheit\"}" }, { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Minneapolis\", \"state\": \"MN\", \"unit\": \"fahrenheit\"}" }, { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Los Angeles\", \"state\": \"CA\", \"unit\": \"fahrenheit\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"NYC\", \"state\": \"NY\", \"unit\": \"fahrenheit\"}" }, { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Baltimore\", \"state\": \"MD\", \"unit\": \"fahrenheit\"}" }, { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Minneapolis\", \"state\": \"MN\", \"unit\": \"fahrenheit\"}" }, { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Los Angeles\", \"state\": \"CA\", \"unit\": \"fahrenheit\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"NYC\", \"state\": \"NY\", \"unit\": \"fahrenheit\"}" }, { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Baltimore\", \"state\": \"MD\", \"unit\": \"fahrenheit\"}" }, { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Minneapolis\", \"state\": \"MN\", \"unit\": \"fahrenheit\"}" }, { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Los Angeles\", \"state\": \"CA\", \"unit\": \"fahrenheit\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"NYC\", \"state\": \"NY\", \"unit\": \"fahrenheit\"}" }, { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Baltimore\", \"state\": \"MD\", \"unit\": \"fahrenheit\"}" }, { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Minneapolis\", \"state\": \"MN\", \"unit\": \"fahrenheit\"}" }, { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Los Angeles\", \"state\": \"CA\", \"unit\": \"fahrenheit\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"NYC\", \"state\": \"NY\", \"unit\": \"fahrenheit\"}" }, { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Baltimore\", \"state\": \"MD\", \"unit\": \"fahrenheit\"}" }, { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Minneapolis\", \"state\": \"MN\", \"unit\": \"fahrenheit\"}" }, { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Los Angeles\", \"state\": \"CA\", \"unit\": \"fahrenheit\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"NYC\", \"state\": \"NY\", \"unit\": \"fahrenheit\"}" }, { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Baltimore\", \"state\": \"MD\", \"unit\": \"fahrenheit\"}" }, { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Minneapolis\", \"state\": \"MN\", \"unit\": \"fahrenheit\"}" }, { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Los Angeles\", \"state\": \"CA\", \"unit\": \"fahrenheit\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"NYC\", \"state\": \"NY\", \"unit\": \"fahrenheit\"}" }, { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Baltimore\", \"state\": \"MD\", \"unit\": \"fahrenheit\"}" }, { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Minneapolis\", \"state\": \"MN\", \"unit\": \"fahrenheit\"}" }, { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Los Angeles\", \"state\": \"CA\", \"unit\": \"fahrenheit\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/vllm-reasoning-multiline", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "qwen3-hermes/vllm-reasoning-without-start-token", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "deepseekv3", "reasoning_parser": "deepseek-r1" }, "model": "Qwen/Qwen3-4B-Thinking-2507", "tokenizer": { "repo": "Qwen/Qwen3-4B-Thinking-2507", "revision": "768f209d9ea81521153ed38c47d515654e938aea", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "3802169b2a02b81e6adb7ab4f64f91ff02db753c8c3a64a01c35192d3a61d8d7", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198, 151667, 198 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "qwen3-hermes/instruct2507-single-call", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Paris\", \"unit\": \"c\"}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Paris\", \"unit\": \"c\"}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Paris\", \"unit\": \"c\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Paris\", \"unit\": \"c\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"c\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": null, "tool_call_detector": "Qwen25Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "Qwen/Qwen3-4B-Instruct-2507", "tokenizer": { "repo": "Qwen/Qwen3-4B-Instruct-2507", "revision": "cdbee75f17c01a7cc42f958dc650907174af0554", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "64f85b198065d0fba2a81f37e10ed68161ce2c19a754c7100e67e0ca2ee9c326", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/instruct2507-parallel-calls", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'search'], got ['get_weather']" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'search'], got ['get_weather']" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'search'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'search'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search'] | rand:1:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search'] | rand:2:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search'] | rand:3:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search'] | rand:4:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search'] | rand:5:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "fail", "detail": "expected 2 calls ['get_weather', 'search'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "special", "status": "fail", "detail": "expected 2 calls ['get_weather', 'search'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": null, "tool_call_detector": "Qwen25Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "Qwen/Qwen3-4B-Instruct-2507", "tokenizer": { "repo": "Qwen/Qwen3-4B-Instruct-2507", "revision": "cdbee75f17c01a7cc42f958dc650907174af0554", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "64f85b198065d0fba2a81f37e10ed68161ce2c19a754c7100e67e0ca2ee9c326", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/instruct2507-no-call", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": null, "tool_call_detector": "Qwen25Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "Qwen/Qwen3-4B-Instruct-2507", "tokenizer": { "repo": "Qwen/Qwen3-4B-Instruct-2507", "revision": "cdbee75f17c01a7cc42f958dc650907174af0554", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "64f85b198065d0fba2a81f37e10ed68161ce2c19a754c7100e67e0ca2ee9c326", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "qwen3-hermes/instruct2507-empty-arguments", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{}' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_time: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_time: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_time: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_time: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": null, "tool_call_detector": "Qwen25Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "Qwen/Qwen3-4B-Instruct-2507", "tokenizer": { "repo": "Qwen/Qwen3-4B-Instruct-2507", "revision": "cdbee75f17c01a7cc42f958dc650907174af0554", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "64f85b198065d0fba2a81f37e10ed68161ce2c19a754c7100e67e0ca2ee9c326", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/instruct2507-text-before-call", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got None; tool_calls[0].arguments: expected '{\"city\": \"London\"}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got None; tool_calls[0].arguments: expected '{\"city\": \"London\"}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got 'Let me check the current'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got 'Let me check the current'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got 'Let me check the current'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got None; tool_calls[0].arguments: nonstream '{\"city\": \"London\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got None; tool_calls[0].arguments: nonstream '{\"city\": \"London\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got 'Let me check the current'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got 'Let me check the current'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got 'Let me check the current conditions in'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Let me check the current conditions in London.', got 'Let me check the current conditions in London.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got 'Let me check the current'" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content: one None, got 'Let me check the current conditions in London.\\n'; tool_calls[0].arguments: one '', got '{\"city\": \"London\"}' | rand:1:8 (fail): content: one None, got 'Let me check the current'; tool_calls[0].arguments: one '', got '{\"city\": \"London\"}' | rand:2:8 (fail): content: one None, got 'Let me check the current'; tool_calls[0].arguments: one '', got '{\"city\": \"London\"}' | rand:3:8 (fail): content: one None, got 'Let me check the current conditions in'; tool_calls[0].arguments: one '', got '{\"city\": \"London\"}' | rand:4:8 (fail): content: one None, got 'Let me check the current conditions in London.\\n'; tool_calls[0].arguments: one '', got '{\"city\": \"London\"}' | rand:5:8 (fail): content: one None, got 'Let me check the current'; tool_calls[0].arguments: one '', got '{\"city\": \"London\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": null, "tool_call_detector": "Qwen25Detector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": null, "reasoning_parser": null }, "model": "Qwen/Qwen3-4B-Instruct-2507", "tokenizer": { "repo": "Qwen/Qwen3-4B-Instruct-2507", "revision": "cdbee75f17c01a7cc42f958dc650907174af0554", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "64f85b198065d0fba2a81f37e10ed68161ce2c19a754c7100e67e0ca2ee9c326", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "Let me check the current conditions in London.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "token": { "content": "Let me check the current conditions in London.\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:1:8": { "content": "Let me check the current", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:2:8": { "content": "Let me check the current", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:3:8": { "content": "Let me check the current conditions in", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:4:8": { "content": "Let me check the current conditions in London.\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:5:8": { "content": "Let me check the current", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/vllm-invalid-json-missing-brace", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "fail", "detail": "returned 1 tool call(s) ['final_answer'] for The call JSON is missing its closing brace and there is no ; no valid call exists." }, { "check": "expected_error", "strategy": "special", "status": "fail", "detail": "returned 1 tool call(s) ['final_answer'] for The call JSON is missing its closing brace and there is no ; no valid call exists." }, { "check": "expected_error", "strategy": "token", "status": "fail", "detail": "returned 1 tool call(s) ['final_answer'] for The call JSON is missing its closing brace and there is no ; no valid call exists." }, { "check": "expected_error", "strategy": "rand:1:8", "status": "fail", "detail": "returned 1 tool call(s) ['final_answer'] for The call JSON is missing its closing brace and there is no ; no valid call exists." }, { "check": "expected_error", "strategy": "rand:2:8", "status": "fail", "detail": "returned 1 tool call(s) ['final_answer'] for The call JSON is missing its closing brace and there is no ; no valid call exists." }, { "check": "expected_error", "strategy": "rand:3:8", "status": "fail", "detail": "returned 1 tool call(s) ['final_answer'] for The call JSON is missing its closing brace and there is no ; no valid call exists." }, { "check": "expected_error", "strategy": "rand:4:8", "status": "fail", "detail": "returned 1 tool call(s) ['final_answer'] for The call JSON is missing its closing brace and there is no ; no valid call exists." }, { "check": "expected_error", "strategy": "rand:5:8", "status": "fail", "detail": "returned 1 tool call(s) ['final_answer'] for The call JSON is missing its closing brace and there is no ; no valid call exists." }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream [], got ['final_answer']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls: nonstream [], got ['final_answer']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "tool_calls: nonstream [], got ['final_answer']" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['final_answer']" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['final_answer']" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['final_answer']" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['final_answer']" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['final_answer']" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{\"trigger\": true}' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"trigger\": true}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{\"trigger\": true}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"trigger\": true}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"trigger\": true}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"trigger\": true}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] final_answer: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] final_answer: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] final_answer: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] final_answer: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 151667, 271, 151668, 271 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "final_answer", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "final_answer", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "final_answer", "arguments_raw": "{\"trigger\": true}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "final_answer", "arguments_raw": "{\"trigger\": true}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "final_answer", "arguments_raw": "{\"trigger\": true}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "final_answer", "arguments_raw": "{\"trigger\": true}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "final_answer", "arguments_raw": "{\"trigger\": true}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "final_answer", "arguments_raw": "{\"trigger\": true}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/sglang-malformed-json-in-tags", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream 'not valid json', got 'not valid json'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream 'not valid json', got 'not valid json'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream 'not valid json', got 'not valid json'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream 'not valid json', got 'not valid json'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream 'not valid json', got 'not valid json'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream 'not valid json', got 'not valid json'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream 'not valid json', got 'not valid json'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream 'not valid json', got 'not valid json'" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 151667, 271, 151668, 271 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "not valid json", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": "not valid json", "reasoning_content": null, "tool_calls": [], "exception": null }, "special": { "content": "not valid json", "reasoning_content": null, "tool_calls": [], "exception": null }, "token": { "content": "not valid json", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:1:8": { "content": "not valid json", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:2:8": { "content": "not valid json", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:3:8": { "content": "not valid json", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:4:8": { "content": "not valid json", "reasoning_content": null, "tool_calls": [], "exception": null }, "rand:5:8": { "content": "not valid json", "reasoning_content": null, "tool_calls": [], "exception": null } } } }, { "fixture_id": "qwen3-hermes/multi-turn-second-call-thinking", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'It is 20C in Zürich. Now search for cafés.', got '\\nIt is 20C in Zürich. Now search for cafés.\\n'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "reasoning_content (whitespace only): expected 'It is 20C in Zürich. Now search for cafés.', got '\\nIt is 20C in Zürich. Now search for cafés.\\n'; tool_calls[0].arguments: expected '{\"query\": \"café Zürich\", \"filters\": {\"open_now\": true}}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "reasoning_content (whitespace only): expected 'It is 20C in Zürich. Now search for cafés.', got '\\nIt is 20C in Zürich. Now search for cafés.\\n'; tool_calls[0].arguments: expected '{\"query\": \"café Zürich\", \"filters\": {\"open_now\": true}}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'It is 20C in Zürich. Now search for cafés.', got '\\nIt is 20C in Zürich. Now search for cafés.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'It is 20C in Zürich. Now search for cafés.', got '\\nIt is 20C in Zürich. Now search for cafés.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'It is 20C in Zürich. Now search for cafés.', got '\\nIt is 20C in Zürich. Now search for cafés.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'It is 20C in Zürich. Now search for cafés.', got '\\nIt is 20C in Zürich. Now search for cafés.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'It is 20C in Zürich. Now search for cafés.', got '\\nIt is 20C in Zürich. Now search for cafés.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'It is 20C in Zürich. Now search for cafés.', got '\\nIt is 20C in Zürich. Now search for cafés.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"query\": \"café Zürich\", \"filters\": {\"open_now\": true}}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"query\": \"café Zürich\", \"filters\": {\"open_now\": true}}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content (whitespace only): one None, got '\\n\\n'; tool_calls[0].arguments: one '', got '{\"query\": \"café Zürich\", \"filters\": {\"open_now\": true}}' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"query\": \"café Zürich\", \"filters\": {\"open_now\": true}}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{\"query\": \"café Zürich\", \"filters\": {\"open_now\": true}}' | rand:3:8 (fail): content (whitespace only): one None, got '\\n\\n'; tool_calls[0].arguments: one '', got '{\"query\": \"café Zürich\", \"filters\": {\"open_now\": true}}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"query\": \"café Zürich\", \"filters\": {\"open_now\": true}}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"query\": \"café Zürich\", \"filters\": {\"open_now\": true}}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] search: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] search: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] search: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] search: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "\nIt is 20C in Zürich. Now search for cafés.\n", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"café Zürich\", \"filters\": {\"open_now\": true}}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "\nIt is 20C in Zürich. Now search for cafés.\n", "tool_calls": [ { "name": "search", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": "\nIt is 20C in Zürich. Now search for cafés.\n", "tool_calls": [ { "name": "search", "arguments_raw": "" } ], "exception": null }, "token": { "content": "\n\n", "reasoning_content": "\nIt is 20C in Zürich. Now search for cafés.\n", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"café Zürich\", \"filters\": {\"open_now\": true}}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "\nIt is 20C in Zürich. Now search for cafés.\n", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"café Zürich\", \"filters\": {\"open_now\": true}}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "\nIt is 20C in Zürich. Now search for cafés.\n", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"café Zürich\", \"filters\": {\"open_now\": true}}" } ], "exception": null }, "rand:3:8": { "content": "\n\n", "reasoning_content": "\nIt is 20C in Zürich. Now search for cafés.\n", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"café Zürich\", \"filters\": {\"open_now\": true}}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "\nIt is 20C in Zürich. Now search for cafés.\n", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"café Zürich\", \"filters\": {\"open_now\": true}}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "\nIt is 20C in Zürich. Now search for cafés.\n", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"café Zürich\", \"filters\": {\"open_now\": true}}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/multi-turn-final-answer-thinking", "family": "qwen3-hermes", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "content (whitespace only): expected 'It is 20 °C in Zürich right now.', got '\\n\\nIt is 20 °C in Zürich right now.'; reasoning_content (whitespace only): expected 'The tool returned 20C.', got '\\nThe tool returned 20C.\\n'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected 'It is 20 °C in Zürich right now.', got '\\n\\nIt is 20 °C in Zürich right now.'; reasoning_content (whitespace only): expected 'The tool returned 20C.', got '\\nThe tool returned 20C.\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected 'It is 20 °C in Zürich right now.', got '\\n\\nIt is 20 °C in Zürich right now.'; reasoning_content (whitespace only): expected 'The tool returned 20C.', got '\\nThe tool returned 20C.\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected 'It is 20 °C in Zürich right now.', got '\\n\\nIt is 20 °C in Zürich right now.'; reasoning_content (whitespace only): expected 'The tool returned 20C.', got '\\nThe tool returned 20C.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'It is 20 °C in Zürich right now.', got '\\n\\nIt is 20 °C in Zürich right now.'; reasoning_content (whitespace only): expected 'The tool returned 20C.', got '\\nThe tool returned 20C.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'It is 20 °C in Zürich right now.', got '\\n\\nIt is 20 °C in Zürich right now.'; reasoning_content (whitespace only): expected 'The tool returned 20C.', got '\\nThe tool returned 20C.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'It is 20 °C in Zürich right now.', got '\\n\\nIt is 20 °C in Zürich right now.'; reasoning_content (whitespace only): expected 'The tool returned 20C.', got '\\nThe tool returned 20C.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'It is 20 °C in Zürich right now.', got '\\n\\nIt is 20 °C in Zürich right now.'; reasoning_content (whitespace only): expected 'The tool returned 20C.', got '\\nThe tool returned 20C.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'It is 20 °C in Zürich right now.', got '\\n\\nIt is 20 °C in Zürich right now.'; reasoning_content (whitespace only): expected 'The tool returned 20C.', got '\\nThe tool returned 20C.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "\n\nIt is 20 °C in Zürich right now.", "reasoning_content": "\nThe tool returned 20C.\n", "tool_calls": [], "exception": null }, "streams": { "one": { "content": "\n\nIt is 20 °C in Zürich right now.", "reasoning_content": "\nThe tool returned 20C.\n", "tool_calls": [], "exception": null }, "special": { "content": "\n\nIt is 20 °C in Zürich right now.", "reasoning_content": "\nThe tool returned 20C.\n", "tool_calls": [], "exception": null }, "token": { "content": "\n\nIt is 20 °C in Zürich right now.", "reasoning_content": "\nThe tool returned 20C.\n", "tool_calls": [], "exception": null }, "rand:1:8": { "content": "\n\nIt is 20 °C in Zürich right now.", "reasoning_content": "\nThe tool returned 20C.\n", "tool_calls": [], "exception": null }, "rand:2:8": { "content": "\n\nIt is 20 °C in Zürich right now.", "reasoning_content": "\nThe tool returned 20C.\n", "tool_calls": [], "exception": null }, "rand:3:8": { "content": "\n\nIt is 20 °C in Zürich right now.", "reasoning_content": "\nThe tool returned 20C.\n", "tool_calls": [], "exception": null }, "rand:4:8": { "content": "\n\nIt is 20 °C in Zürich right now.", "reasoning_content": "\nThe tool returned 20C.\n", "tool_calls": [], "exception": null }, "rand:5:8": { "content": "\n\nIt is 20 °C in Zürich right now.", "reasoning_content": "\nThe tool returned 20C.\n", "tool_calls": [], "exception": null } } } }, { "fixture_id": "qwen3-hermes/parallel-two-calls-thinking", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got '\\nI should call the tools.\\n'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got '\\nI should call the tools.\\n'; tool_calls: expected ['get_weather', 'search'], got ['get_weather']" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got '\\nI should call the tools.\\n'; tool_calls: expected ['get_weather', 'search'], got ['get_weather']" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'I should call the tools.', got '\\nI should call the tools.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'I should call the tools.', got '\\nI should call the tools.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got '\\nI should call the tools.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got '\\nI should call the tools.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got '\\nI should call the tools.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got '\\nI should call the tools.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'search'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'search'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content (whitespace only): one None, got '\\n\\n'; tool_calls: one ['get_weather'], got ['get_weather', 'search'] | rand:1:8 (fail): content (whitespace only): one None, got '\\n\\n'; tool_calls: one ['get_weather'], got ['get_weather', 'search'] | rand:2:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search'] | rand:3:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search'] | rand:4:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search'] | rand:5:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "fail", "detail": "expected 2 calls ['get_weather', 'search'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "special", "status": "fail", "detail": "expected 2 calls ['get_weather', 'search'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "\nI should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "\nI should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": "\nI should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "token": { "content": "\n\n", "reasoning_content": "\nI should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:1:8": { "content": "\n\n", "reasoning_content": "\nI should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "\nI should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "\nI should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "\nI should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "\nI should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/parallel-three-calls-no-thinking", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_weather', 'get_weather'], got ['get_weather']" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls: expected ['get_weather', 'get_weather', 'get_weather'], got ['get_weather']" }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'get_weather', 'get_weather'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'get_weather', 'get_weather'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls: one ['get_weather'], got ['get_weather', 'get_weather', 'get_weather'] | rand:1:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'get_weather', 'get_weather'] | rand:2:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'get_weather', 'get_weather'] | rand:3:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'get_weather', 'get_weather'] | rand:4:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'get_weather', 'get_weather'] | rand:5:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'get_weather', 'get_weather']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "fail", "detail": "expected 3 calls ['get_weather', 'get_weather', 'get_weather'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "special", "status": "fail", "detail": "expected 3 calls ['get_weather', 'get_weather', 'get_weather'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 151667, 271, 151668, 271 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"c\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"c\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"c\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"c\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"c\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"c\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"c\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/parallel-mixed-tools-text-before-no-thinking", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected \"I'll fetch both for you.\", got None; tool_calls: expected ['get_time', 'get_weather'], got ['get_time']" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected \"I'll fetch both for you.\", got None; tool_calls: expected ['get_time', 'get_weather'], got ['get_time']" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected \"I'll fetch both for you.\", got \"I'll fetch both for you.\\n\"" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected \"I'll fetch both for you.\", got \"I'll fetch both for\"" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected \"I'll fetch both for you.\", got \"I'll fetch both for\"" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected \"I'll fetch both for you.\", got \"I'll fetch both for you.\\n\"" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected \"I'll fetch both for you.\", got \"I'll fetch both\"" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected \"I'll fetch both for you.\", got \"I'll fetch both for\"" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream \"I'll fetch both for you.\", got None; tool_calls: nonstream ['get_time', 'get_weather'], got ['get_time']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream \"I'll fetch both for you.\", got None; tool_calls: nonstream ['get_time', 'get_weather'], got ['get_time']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream \"I'll fetch both for you.\", got \"I'll fetch both for you.\\n\"" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream \"I'll fetch both for you.\", got \"I'll fetch both for\"" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream \"I'll fetch both for you.\", got \"I'll fetch both for\"" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream \"I'll fetch both for you.\", got \"I'll fetch both for you.\\n\"" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream \"I'll fetch both for you.\", got \"I'll fetch both\"" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream \"I'll fetch both for you.\", got \"I'll fetch both for\"" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content: one None, got \"I'll fetch both for you.\\n\"; tool_calls: one ['get_time'], got ['get_time', 'get_weather'] | rand:1:8 (fail): content: one None, got \"I'll fetch both for\"; tool_calls: one ['get_time'], got ['get_time', 'get_weather'] | rand:2:8 (fail): content: one None, got \"I'll fetch both for\"; tool_calls: one ['get_time'], got ['get_time', 'get_weather'] | rand:3:8 (fail): content: one None, got \"I'll fetch both for you.\\n\"; tool_calls: one ['get_time'], got ['get_time', 'get_weather'] | rand:4:8 (fail): content: one None, got \"I'll fetch both\"; tool_calls: one ['get_time'], got ['get_time', 'get_weather'] | rand:5:8 (fail): content: one None, got \"I'll fetch both for\"; tool_calls: one ['get_time'], got ['get_time', 'get_weather']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_time: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_time: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_time: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_time: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "fail", "detail": "expected 2 calls ['get_time', 'get_weather'], got 1 ['get_time']" }, { "check": "parallel_order", "strategy": "special", "status": "fail", "detail": "expected 2 calls ['get_time', 'get_weather'], got 1 ['get_time']" }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 151667, 271, 151668, 271 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "I'll fetch both for you.", "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\", \"unit\": \"c\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "" } ], "exception": null }, "token": { "content": "I'll fetch both for you.\n", "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\", \"unit\": \"c\"}" } ], "exception": null }, "rand:1:8": { "content": "I'll fetch both for", "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\", \"unit\": \"c\"}" } ], "exception": null }, "rand:2:8": { "content": "I'll fetch both for", "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\", \"unit\": \"c\"}" } ], "exception": null }, "rand:3:8": { "content": "I'll fetch both for you.\n", "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\", \"unit\": \"c\"}" } ], "exception": null }, "rand:4:8": { "content": "I'll fetch both", "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\", \"unit\": \"c\"}" } ], "exception": null }, "rand:5:8": { "content": "I'll fetch both for", "reasoning_content": null, "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\", \"unit\": \"c\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/thinking2507-single-call", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"; tool_calls[0].arguments: expected '{\"city\": \"Paris\", \"unit\": \"c\"}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"; tool_calls[0].arguments: expected '{\"city\": \"Paris\", \"unit\": \"c\"}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Paris\", \"unit\": \"c\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Paris\", \"unit\": \"c\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content (whitespace only): one None, got '\\n\\n'; tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"c\"}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\", \"unit\": \"c\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "deepseekv3", "reasoning_parser": "deepseek-r1" }, "model": "Qwen/Qwen3-4B-Thinking-2507", "tokenizer": { "repo": "Qwen/Qwen3-4B-Thinking-2507", "revision": "768f209d9ea81521153ed38c47d515654e938aea", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "3802169b2a02b81e6adb7ab4f64f91ff02db753c8c3a64a01c35192d3a61d8d7", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198, 151667, 198 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "token": { "content": "\n\n", "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/thinking2507-parallel-calls", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'; tool_calls: expected ['get_weather', 'search'], got ['get_weather']" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'; tool_calls: expected ['get_weather', 'search'], got ['get_weather']" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'search'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls: nonstream ['get_weather', 'search'], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content (whitespace only): one None, got '\\n\\n'; tool_calls: one ['get_weather'], got ['get_weather', 'search'] | rand:1:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search'] | rand:2:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search'] | rand:3:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search'] | rand:4:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search'] | rand:5:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "fail", "detail": "expected 2 calls ['get_weather', 'search'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "special", "status": "fail", "detail": "expected 2 calls ['get_weather', 'search'], got 1 ['get_weather']" }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "deepseekv3", "reasoning_parser": "deepseek-r1" }, "model": "Qwen/Qwen3-4B-Thinking-2507", "tokenizer": { "repo": "Qwen/Qwen3-4B-Thinking-2507", "revision": "768f209d9ea81521153ed38c47d515654e938aea", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "3802169b2a02b81e6adb7ab4f64f91ff02db753c8c3a64a01c35192d3a61d8d7", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198, 151667, 198 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "token": { "content": "\n\n", "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/thinking2507-no-call", "family": "qwen3-hermes", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "content (whitespace only): expected '2 + 2 = 4.', got '\\n\\n2 + 2 = 4.'; reasoning_content (whitespace only): expected 'Simple arithmetic, no tool needed.', got 'Simple arithmetic, no tool needed.\\n'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected '2 + 2 = 4.', got '\\n\\n2 + 2 = 4.'; reasoning_content (whitespace only): expected 'Simple arithmetic, no tool needed.', got 'Simple arithmetic, no tool needed.\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected '2 + 2 = 4.', got '\\n\\n2 + 2 = 4.'; reasoning_content (whitespace only): expected 'Simple arithmetic, no tool needed.', got 'Simple arithmetic, no tool needed.\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected '2 + 2 = 4.', got '\\n\\n2 + 2 = 4.'; reasoning_content (whitespace only): expected 'Simple arithmetic, no tool needed.', got 'Simple arithmetic, no tool needed.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected '2 + 2 = 4.', got '\\n\\n2 + 2 = 4.'; reasoning_content (whitespace only): expected 'Simple arithmetic, no tool needed.', got 'Simple arithmetic, no tool needed.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected '2 + 2 = 4.', got '\\n\\n2 + 2 = 4.'; reasoning_content (whitespace only): expected 'Simple arithmetic, no tool needed.', got 'Simple arithmetic, no tool needed.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected '2 + 2 = 4.', got '\\n\\n2 + 2 = 4.'; reasoning_content (whitespace only): expected 'Simple arithmetic, no tool needed.', got 'Simple arithmetic, no tool needed.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected '2 + 2 = 4.', got '\\n\\n2 + 2 = 4.'; reasoning_content (whitespace only): expected 'Simple arithmetic, no tool needed.', got 'Simple arithmetic, no tool needed.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected '2 + 2 = 4.', got '\\n\\n2 + 2 = 4.'; reasoning_content (whitespace only): expected 'Simple arithmetic, no tool needed.', got 'Simple arithmetic, no tool needed.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "deepseekv3", "reasoning_parser": "deepseek-r1" }, "model": "Qwen/Qwen3-4B-Thinking-2507", "tokenizer": { "repo": "Qwen/Qwen3-4B-Thinking-2507", "revision": "768f209d9ea81521153ed38c47d515654e938aea", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "3802169b2a02b81e6adb7ab4f64f91ff02db753c8c3a64a01c35192d3a61d8d7", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198, 151667, 198 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "\n\n2 + 2 = 4.", "reasoning_content": "Simple arithmetic, no tool needed.\n", "tool_calls": [], "exception": null }, "streams": { "one": { "content": "\n\n2 + 2 = 4.", "reasoning_content": "Simple arithmetic, no tool needed.\n", "tool_calls": [], "exception": null }, "special": { "content": "\n\n2 + 2 = 4.", "reasoning_content": "Simple arithmetic, no tool needed.\n", "tool_calls": [], "exception": null }, "token": { "content": "\n\n2 + 2 = 4.", "reasoning_content": "Simple arithmetic, no tool needed.\n", "tool_calls": [], "exception": null }, "rand:1:8": { "content": "\n\n2 + 2 = 4.", "reasoning_content": "Simple arithmetic, no tool needed.\n", "tool_calls": [], "exception": null }, "rand:2:8": { "content": "\n\n2 + 2 = 4.", "reasoning_content": "Simple arithmetic, no tool needed.\n", "tool_calls": [], "exception": null }, "rand:3:8": { "content": "\n\n2 + 2 = 4.", "reasoning_content": "Simple arithmetic, no tool needed.\n", "tool_calls": [], "exception": null }, "rand:4:8": { "content": "\n\n2 + 2 = 4.", "reasoning_content": "Simple arithmetic, no tool needed.\n", "tool_calls": [], "exception": null }, "rand:5:8": { "content": "\n\n2 + 2 = 4.", "reasoning_content": "Simple arithmetic, no tool needed.\n", "tool_calls": [], "exception": null } } } }, { "fixture_id": "qwen3-hermes/thinking2507-text-before-call", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I need live weather data for London.', got 'I need live weather data for London.\\n'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got None; reasoning_content (whitespace only): expected 'I need live weather data for London.', got 'I need live weather data for London.\\n'; tool_calls[0].arguments: expected '{\"city\": \"London\"}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got None; reasoning_content (whitespace only): expected 'I need live weather data for London.', got 'I need live weather data for London.\\n'; tool_calls[0].arguments: expected '{\"city\": \"London\"}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got '\\n\\nLet me check the current conditions in London.\\n'; reasoning_content (whitespace only): expected 'I need live weather data for London.', got 'I need live weather data for London.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got '\\n\\nLet me'; reasoning_content (whitespace only): expected 'I need live weather data for London.', got 'I need live weather data for London.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the current conditions in London.', got '\\n\\nLet me check the current conditions in London.\\n'; reasoning_content (whitespace only): expected 'I need live weather data for London.', got 'I need live weather data for London.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got '\\n\\nLet me check'; reasoning_content (whitespace only): expected 'I need live weather data for London.', got 'I need live weather data for London.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got '\\n\\nLet me check the current conditions in London'; reasoning_content (whitespace only): expected 'I need live weather data for London.', got 'I need live weather data for London.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected 'Let me check the current conditions in London.', got '\\n\\nLet me'; reasoning_content (whitespace only): expected 'I need live weather data for London.', got 'I need live weather data for London.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got None; tool_calls[0].arguments: nonstream '{\"city\": \"London\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got None; tool_calls[0].arguments: nonstream '{\"city\": \"London\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Let me check the current conditions in London.', got '\\n\\nLet me check the current conditions in London.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got '\\n\\nLet me'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Let me check the current conditions in London.', got '\\n\\nLet me check the current conditions in London.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got '\\n\\nLet me check'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got '\\n\\nLet me check the current conditions in London'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream 'Let me check the current conditions in London.', got '\\n\\nLet me'" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content: one None, got '\\n\\nLet me check the current conditions in London.\\n'; tool_calls[0].arguments: one '', got '{\"city\": \"London\"}' | rand:1:8 (fail): content: one None, got '\\n\\nLet me'; tool_calls[0].arguments: one '', got '{\"city\": \"London\"}' | rand:2:8 (fail): content: one None, got '\\n\\nLet me check the current conditions in London.\\n'; tool_calls[0].arguments: one '', got '{\"city\": \"London\"}' | rand:3:8 (fail): content: one None, got '\\n\\nLet me check'; tool_calls[0].arguments: one '', got '{\"city\": \"London\"}' | rand:4:8 (fail): content: one None, got '\\n\\nLet me check the current conditions in London'; tool_calls[0].arguments: one '', got '{\"city\": \"London\"}' | rand:5:8 (fail): content: one None, got '\\n\\nLet me'; tool_calls[0].arguments: one '', got '{\"city\": \"London\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "deepseekv3", "reasoning_parser": "deepseek-r1" }, "model": "Qwen/Qwen3-4B-Thinking-2507", "tokenizer": { "repo": "Qwen/Qwen3-4B-Thinking-2507", "revision": "768f209d9ea81521153ed38c47d515654e938aea", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "3802169b2a02b81e6adb7ab4f64f91ff02db753c8c3a64a01c35192d3a61d8d7", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198, 151667, 198 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "Let me check the current conditions in London.", "reasoning_content": "I need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "I need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": "I need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "token": { "content": "\n\nLet me check the current conditions in London.\n", "reasoning_content": "I need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:1:8": { "content": "\n\nLet me", "reasoning_content": "I need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:2:8": { "content": "\n\nLet me check the current conditions in London.\n", "reasoning_content": "I need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:3:8": { "content": "\n\nLet me check", "reasoning_content": "I need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:4:8": { "content": "\n\nLet me check the current conditions in London", "reasoning_content": "I need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:5:8": { "content": "\n\nLet me", "reasoning_content": "I need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/thinking2507-multi-turn", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'It is 20C. Now the café search.', got 'It is 20C. Now the café search.\\n'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "reasoning_content (whitespace only): expected 'It is 20C. Now the café search.', got 'It is 20C. Now the café search.\\n'; tool_calls[0].arguments: expected '{\"query\": \"café Zürich\"}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "reasoning_content (whitespace only): expected 'It is 20C. Now the café search.', got 'It is 20C. Now the café search.\\n'; tool_calls[0].arguments: expected '{\"query\": \"café Zürich\"}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'It is 20C. Now the café search.', got 'It is 20C. Now the café search.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'It is 20C. Now the café search.', got 'It is 20C. Now the café search.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'It is 20C. Now the café search.', got 'It is 20C. Now the café search.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'It is 20C. Now the café search.', got 'It is 20C. Now the café search.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'It is 20C. Now the café search.', got 'It is 20C. Now the café search.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'It is 20C. Now the café search.', got 'It is 20C. Now the café search.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"query\": \"café Zürich\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"query\": \"café Zürich\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content (whitespace only): one None, got '\\n\\n'; tool_calls[0].arguments: one '', got '{\"query\": \"café Zürich\"}' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"query\": \"café Zürich\"}' | rand:2:8 (fail): content (whitespace only): one None, got '\\n\\n'; tool_calls[0].arguments: one '', got '{\"query\": \"café Zürich\"}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"query\": \"café Zürich\"}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"query\": \"café Zürich\"}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"query\": \"café Zürich\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] search: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] search: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] search: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] search: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "deepseekv3", "reasoning_parser": "deepseek-r1" }, "model": "Qwen/Qwen3-4B-Thinking-2507", "tokenizer": { "repo": "Qwen/Qwen3-4B-Thinking-2507", "revision": "768f209d9ea81521153ed38c47d515654e938aea", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "3802169b2a02b81e6adb7ab4f64f91ff02db753c8c3a64a01c35192d3a61d8d7", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198, 151667, 198 ] }, "stop": { "token": "<|endoftext|>", "id": 151643, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "It is 20C. Now the café search.\n", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"café Zürich\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "It is 20C. Now the café search.\n", "tool_calls": [ { "name": "search", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": "It is 20C. Now the café search.\n", "tool_calls": [ { "name": "search", "arguments_raw": "" } ], "exception": null }, "token": { "content": "\n\n", "reasoning_content": "It is 20C. Now the café search.\n", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"café Zürich\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "It is 20C. Now the café search.\n", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"café Zürich\"}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": "It is 20C. Now the café search.\n", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"café Zürich\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "It is 20C. Now the café search.\n", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"café Zürich\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "It is 20C. Now the café search.\n", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"café Zürich\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "It is 20C. Now the café search.\n", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"café Zürich\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/truncated-mid-arguments-no-thinking", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool-call JSON: no complete call exists." }, { "check": "expected_error", "strategy": "special", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool-call JSON: no complete call exists." }, { "check": "expected_error", "strategy": "token", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool-call JSON: no complete call exists." }, { "check": "expected_error", "strategy": "rand:1:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool-call JSON: no complete call exists." }, { "check": "expected_error", "strategy": "rand:2:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool-call JSON: no complete call exists." }, { "check": "expected_error", "strategy": "rand:3:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool-call JSON: no complete call exists." }, { "check": "expected_error", "strategy": "rand:4:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool-call JSON: no complete call exists." }, { "check": "expected_error", "strategy": "rand:5:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool-call JSON: no complete call exists." }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\"}' | rand:1:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\"}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\"}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\"}' | rand:4:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\"}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{\"city\": \"Paris\"}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 151667, 271, 151668, 271 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/truncated-mid-arguments-thinking", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool-call JSON: no complete call exists." }, { "check": "expected_error", "strategy": "special", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool-call JSON: no complete call exists." }, { "check": "expected_error", "strategy": "token", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool-call JSON: no complete call exists." }, { "check": "expected_error", "strategy": "rand:1:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool-call JSON: no complete call exists." }, { "check": "expected_error", "strategy": "rand:2:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool-call JSON: no complete call exists." }, { "check": "expected_error", "strategy": "rand:3:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool-call JSON: no complete call exists." }, { "check": "expected_error", "strategy": "rand:4:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool-call JSON: no complete call exists." }, { "check": "expected_error", "strategy": "rand:5:8", "status": "fail", "detail": "returned 1 tool call(s) ['get_weather'] for Output cut by max_tokens inside the tool-call JSON: no complete call exists." }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content (whitespace only): nonstream None, got '\\n\\n'; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content (whitespace only): nonstream None, got '\\n\\n'; tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): content (whitespace only): one None, got '\\n\\n'; tool_calls[0].arguments: one '', got '{}' | rand:1:8 (fail): content (whitespace only): one None, got '\\n\\n'; tool_calls[0].arguments: one '', got '{}' | rand:2:8 (fail): tool_calls[0].arguments: one '', got '{}' | rand:3:8 (fail): tool_calls[0].arguments: one '', got '{}' | rand:5:8 (fail): tool_calls[0].arguments: one '', got '{}'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "fail", "detail": "[0] get_weather: $: 'city' is a required property" }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "fail", "detail": "[0] get_weather: $: 'city' is a required property" }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "fail", "detail": "[0] get_weather: $: 'city' is a required property" }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "fail", "detail": "[0] get_weather: $: 'city' is a required property" }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "fail", "detail": "[0] get_weather: $: 'city' is a required property" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "\nI should call the tool.\n", "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "\nI should call the tool.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": "\nI should call the tool.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "token": { "content": "\n\n", "reasoning_content": "\nI should call the tool.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{}" } ], "exception": null }, "rand:1:8": { "content": "\n\n", "reasoning_content": "\nI should call the tool.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "\nI should call the tool.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "\nI should call the tool.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "\nI should call the tool.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "\nI should call the tool.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/truncated-after-open-tag", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 151667, 271, 151668, 271 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "qwen3-hermes/truncated-second-parallel-call", "family": "qwen3-hermes", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Zürich\", \"unit\": \"c\"}', got ''" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"city\": \"Zürich\", \"unit\": \"c\"}', got ''" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: expected ['get_weather'], got ['get_weather', 'search']" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Zürich\", \"unit\": \"c\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"city\": \"Zürich\", \"unit\": \"c\"}', got ''" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "tool_calls: nonstream ['get_weather'], got ['get_weather', 'search']" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: nonstream ['get_weather'], got ['get_weather', 'search']" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: nonstream ['get_weather'], got ['get_weather', 'search']" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: nonstream ['get_weather'], got ['get_weather', 'search']" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: nonstream ['get_weather'], got ['get_weather', 'search']" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: nonstream ['get_weather'], got ['get_weather', 'search']" }, { "check": "split_invariance", "strategy": "*", "status": "fail", "detail": "vs one: token (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search'] | rand:1:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search'] | rand:2:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search'] | rand:3:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search'] | rand:4:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search'] | rand:5:8 (fail): tool_calls: one ['get_weather'], got ['get_weather', 'search']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not valid JSON text ('': Expecting value: line 1 column 1 (char 0))" }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] get_weather: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 151667, 271, 151668, 271 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "" } ], "exception": null }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-hermes/truncated-in-reasoning", "family": "qwen3-hermes", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user wants the current weather', got '\\nThe user wants the current weather'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user wants the current weather', got '\\nThe user wants the current weather'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user wants the current weather', got '\\nThe user wants the current weather'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user wants the current weather', got '\\nThe user wants the current weather'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user wants the current weather', got '\\nThe user wants the current weather'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user wants the current weather', got '\\nThe user wants the current weather'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user wants the current weather', got '\\nThe user wants the current weather'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user wants the current weather', got '\\nThe user wants the current weather'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user wants the current weather', got '\\nThe user wants the current weather'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3-0.6B", "tokenizer": { "repo": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "a55ee1b1660128b7098723e0abcd92caa0788061051c62d51cbe87d9cf1974d8", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": true, "chat_template_kwargs": { "enable_thinking": true }, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "\nThe user wants the current weather", "tool_calls": [], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "\nThe user wants the current weather", "tool_calls": [], "exception": null }, "special": { "content": null, "reasoning_content": "\nThe user wants the current weather", "tool_calls": [], "exception": null }, "token": { "content": null, "reasoning_content": "\nThe user wants the current weather", "tool_calls": [], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "\nThe user wants the current weather", "tool_calls": [], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "\nThe user wants the current weather", "tool_calls": [], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "\nThe user wants the current weather", "tool_calls": [], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "\nThe user wants the current weather", "tool_calls": [], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "\nThe user wants the current weather", "tool_calls": [], "exception": null } } } }, { "fixture_id": "qwen3-hermes/thinking2507-truncated-in-reasoning", "family": "qwen3-hermes", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen25", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen25Detector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "deepseekv3", "reasoning_parser": "deepseek-r1" }, "model": "Qwen/Qwen3-4B-Thinking-2507", "tokenizer": { "repo": "Qwen/Qwen3-4B-Thinking-2507", "revision": "768f209d9ea81521153ed38c47d515654e938aea", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3ForCausalLM" ], "model_type": "qwen3" }, "hf_config_error": null, "chat_template_sha256": "3802169b2a02b81e6adb7ab4f64f91ff02db753c8c3a64a01c35192d3a61d8d7", "template_force_reasoning": true, "template_reasoning_config": "ReasoningToggleConfig(toggle_param=None, default_enabled=None, special_case='always', effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198, 151667, 198 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "qwen3-xml/llamacpp-q35-reasoning-then-call", "family": "qwen3-xml", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected \"I'm\\nthinking\", got \"I'm\\nthinking\\n\"" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected \"I'm\\nthinking\", got \"I'm\\nthinking\\n\"" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected \"I'm\\nthinking\", got \"I'm\\nthinking\\n\"" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected \"I'm\\nthinking\", got \"I'm\\nthinking\\n\"" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected \"I'm\\nthinking\", got \"I'm\\nthinking\\n\"" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected \"I'm\\nthinking\", got \"I'm\\nthinking\\n\"" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected \"I'm\\nthinking\", got \"I'm\\nthinking\\n\"" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected \"I'm\\nthinking\", got \"I'm\\nthinking\\n\"" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected \"I'm\\nthinking\", got \"I'm\\nthinking\\n\"" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.5-9B", "tokenizer": { "repo": "Qwen/Qwen3.5-9B", "revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 248045, 74455, 198, 248068, 198 ] }, "stop": { "token": "<|im_end|>", "id": 248046, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "\n\n", "reasoning_content": "I'm\nthinking\n", "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "streams": { "one": { "content": "\n\n", "reasoning_content": "I'm\nthinking\n", "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "special": { "content": "\n\n", "reasoning_content": "I'm\nthinking\n", "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "token": { "content": "\n\n", "reasoning_content": "I'm\nthinking\n", "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "rand:1:8": { "content": "\n\n", "reasoning_content": "I'm\nthinking\n", "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": "I'm\nthinking\n", "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "rand:3:8": { "content": "\n\n", "reasoning_content": "I'm\nthinking\n", "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "rand:4:8": { "content": "\n\n", "reasoning_content": "I'm\nthinking\n", "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "rand:5:8": { "content": "\n\n", "reasoning_content": "I'm\nthinking\n", "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/llamacpp-q35-parallel-no-think", "family": "qwen3-xml", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.5-9B", "tokenizer": { "repo": "Qwen/Qwen3.5-9B", "revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 248068, 271, 248069, 271 ] }, "stop": { "token": "<|im_end|>", "id": 248046, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\": 1, \"arg2\": 2}" } ], "exception": null }, "streams": { "one": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\": 1, \"arg2\": 2}" } ], "exception": null }, "special": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\": 1, \"arg2\": 2}" } ], "exception": null }, "token": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\": 1, \"arg2\": 2}" } ], "exception": null }, "rand:1:8": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\": 1, \"arg2\": 2}" } ], "exception": null }, "rand:2:8": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\": 1, \"arg2\": 2}" } ], "exception": null }, "rand:3:8": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\": 1, \"arg2\": 2}" } ], "exception": null }, "rand:4:8": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\": 1, \"arg2\": 2}" } ], "exception": null }, "rand:5:8": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\": 1, \"arg2\": 2}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/llamacpp-q35-python-multiline", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.5-9B", "tokenizer": { "repo": "Qwen/Qwen3.5-9B", "revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 248068, 271, 248069, 271 ] }, "stop": { "token": "<|im_end|>", "id": 248046, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "qwen3-xml/llamacpp-q35-value-trailing-newline", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.5-9B", "tokenizer": { "repo": "Qwen/Qwen3.5-9B", "revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 248068, 271, 248069, 271 ] }, "stop": { "token": "<|im_end|>", "id": 248046, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "qwen3-xml/llamacpp-q35-code-leading-indent", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.5-9B", "tokenizer": { "repo": "Qwen/Qwen3.5-9B", "revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 248068, 271, 248069, 271 ] }, "stop": { "token": "<|im_end|>", "id": 248046, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "qwen3-xml/llamacpp-q35-call-inside-think", "family": "qwen3-xml", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Need to inspect the current directory.', got 'Need to inspect the current directory.\\n'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Need to inspect the current directory.', got 'Need to inspect the current directory.\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Need to inspect the current directory.', got 'Need to inspect the current directory.\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Need to inspect the current directory.', got 'Need to inspect the current directory.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Need to inspect the current directory.', got 'Need to inspect the current directory.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Need to inspect the current directory.', got 'Need to inspect the current directory.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Need to inspect the current directory.', got 'Need to inspect the current directory.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Need to inspect the current directory.', got 'Need to inspect the current directory.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'Need to inspect the current directory.', got 'Need to inspect the current directory.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.5-9B", "tokenizer": { "repo": "Qwen/Qwen3.5-9B", "revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 248045, 74455, 198, 248068, 198 ] }, "stop": { "token": "<|im_end|>", "id": 248046, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": "Need to inspect the current directory.\n", "tool_calls": [ { "name": "run_in_terminal", "arguments_raw": "{\"command\": \"pwd\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "Need to inspect the current directory.\n", "tool_calls": [ { "name": "run_in_terminal", "arguments_raw": "{\"command\": \"pwd\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "Need to inspect the current directory.\n", "tool_calls": [ { "name": "run_in_terminal", "arguments_raw": "{\"command\": \"pwd\"}" } ], "exception": null }, "token": { "content": null, "reasoning_content": "Need to inspect the current directory.\n", "tool_calls": [ { "name": "run_in_terminal", "arguments_raw": "{\"command\": \"pwd\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "Need to inspect the current directory.\n", "tool_calls": [ { "name": "run_in_terminal", "arguments_raw": "{\"command\": \"pwd\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "Need to inspect the current directory.\n", "tool_calls": [ { "name": "run_in_terminal", "arguments_raw": "{\"command\": \"pwd\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "Need to inspect the current directory.\n", "tool_calls": [ { "name": "run_in_terminal", "arguments_raw": "{\"command\": \"pwd\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "Need to inspect the current directory.\n", "tool_calls": [ { "name": "run_in_terminal", "arguments_raw": "{\"command\": \"pwd\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "Need to inspect the current directory.\n", "tool_calls": [ { "name": "run_in_terminal", "arguments_raw": "{\"command\": \"pwd\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/llamacpp-q35-empty-args", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.5-9B", "tokenizer": { "repo": "Qwen/Qwen3.5-9B", "revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 248068, 271, 248069, 271 ] }, "stop": { "token": "<|im_end|>", "id": 248046, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "qwen3-xml/llamacpp-q35-reasoning-content-call", "family": "qwen3-xml", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me inspect it now.', got '\\n\\nLet me inspect it now.\\n'; reasoning_content (whitespace only): expected 'I should inspect the directory.', got 'I should inspect the directory.\\n'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me inspect it now.', got '\\n\\nLet me inspect it now.\\n'; reasoning_content (whitespace only): expected 'I should inspect the directory.', got 'I should inspect the directory.\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me inspect it now.', got '\\n\\nLet me inspect it now.\\n'; reasoning_content (whitespace only): expected 'I should inspect the directory.', got 'I should inspect the directory.\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me inspect it now.', got '\\n\\nLet me inspect it now.\\n'; reasoning_content (whitespace only): expected 'I should inspect the directory.', got 'I should inspect the directory.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me inspect it now.', got '\\n\\nLet me inspect it now.\\n'; reasoning_content (whitespace only): expected 'I should inspect the directory.', got 'I should inspect the directory.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me inspect it now.', got '\\n\\nLet me inspect it now.\\n'; reasoning_content (whitespace only): expected 'I should inspect the directory.', got 'I should inspect the directory.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me inspect it now.', got '\\n\\nLet me inspect it now.\\n'; reasoning_content (whitespace only): expected 'I should inspect the directory.', got 'I should inspect the directory.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me inspect it now.', got '\\n\\nLet me inspect it now.\\n'; reasoning_content (whitespace only): expected 'I should inspect the directory.', got 'I should inspect the directory.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me inspect it now.', got '\\n\\nLet me inspect it now.\\n'; reasoning_content (whitespace only): expected 'I should inspect the directory.', got 'I should inspect the directory.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.5-9B", "tokenizer": { "repo": "Qwen/Qwen3.5-9B", "revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 248045, 74455, 198, 248068, 198 ] }, "stop": { "token": "<|im_end|>", "id": 248046, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "\n\nLet me inspect it now.\n", "reasoning_content": "I should inspect the directory.\n", "tool_calls": [ { "name": "run_in_terminal", "arguments_raw": "{\"command\": \"pwd\"}" } ], "exception": null }, "streams": { "one": { "content": "\n\nLet me inspect it now.\n", "reasoning_content": "I should inspect the directory.\n", "tool_calls": [ { "name": "run_in_terminal", "arguments_raw": "{\"command\": \"pwd\"}" } ], "exception": null }, "special": { "content": "\n\nLet me inspect it now.\n", "reasoning_content": "I should inspect the directory.\n", "tool_calls": [ { "name": "run_in_terminal", "arguments_raw": "{\"command\": \"pwd\"}" } ], "exception": null }, "token": { "content": "\n\nLet me inspect it now.\n", "reasoning_content": "I should inspect the directory.\n", "tool_calls": [ { "name": "run_in_terminal", "arguments_raw": "{\"command\": \"pwd\"}" } ], "exception": null }, "rand:1:8": { "content": "\n\nLet me inspect it now.\n", "reasoning_content": "I should inspect the directory.\n", "tool_calls": [ { "name": "run_in_terminal", "arguments_raw": "{\"command\": \"pwd\"}" } ], "exception": null }, "rand:2:8": { "content": "\n\nLet me inspect it now.\n", "reasoning_content": "I should inspect the directory.\n", "tool_calls": [ { "name": "run_in_terminal", "arguments_raw": "{\"command\": \"pwd\"}" } ], "exception": null }, "rand:3:8": { "content": "\n\nLet me inspect it now.\n", "reasoning_content": "I should inspect the directory.\n", "tool_calls": [ { "name": "run_in_terminal", "arguments_raw": "{\"command\": \"pwd\"}" } ], "exception": null }, "rand:4:8": { "content": "\n\nLet me inspect it now.\n", "reasoning_content": "I should inspect the directory.\n", "tool_calls": [ { "name": "run_in_terminal", "arguments_raw": "{\"command\": \"pwd\"}" } ], "exception": null }, "rand:5:8": { "content": "\n\nLet me inspect it now.\n", "reasoning_content": "I should inspect the directory.\n", "tool_calls": [ { "name": "run_in_terminal", "arguments_raw": "{\"command\": \"pwd\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/llamacpp-coder-bare-function", "family": "qwen3-xml", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '\\n\\n1\\n\\n\\n'; tool_calls: expected ['special_function'], got []" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '\\n\\n1\\n\\n\\n', got '\\n\\n\\n'; tool_calls: nonstream [], got ['special_function']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '\\n\\n1\\n\\n\\n', got '\\n\\n\\n'; tool_calls: nonstream [], got ['special_function']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '\\n\\n1\\n\\n\\n', got '\\n\\n\\n'; tool_calls: nonstream [], got ['special_function']" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '\\n\\n1\\n\\n\\n', got '\\n\\n\\n'; tool_calls: nonstream [], got ['special_function']" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '\\n\\n1\\n\\n\\n', got '\\n\\n\\n'; tool_calls: nonstream [], got ['special_function']" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '\\n\\n1\\n\\n\\n', got '\\n\\n\\n'; tool_calls: nonstream [], got ['special_function']" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '\\n\\n1\\n\\n\\n', got '\\n\\n\\n'; tool_calls: nonstream [], got ['special_function']" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '\\n\\n1\\n\\n\\n', got '\\n\\n\\n'; tool_calls: nonstream [], got ['special_function']" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": null, "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": null }, "model": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "tokenizer": { "repo": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3MoeForCausalLM" ], "model_type": "qwen3_moe" }, "hf_config_error": null, "chat_template_sha256": "5a38bfa05833266240066aedc497decc9b00cc0d3e3b8cceea98cf530196ab06", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|im_end|>", "id": 151645, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "\n\n1\n\n\n", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": "\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "special": { "content": "\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "token": { "content": "\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "rand:1:8": { "content": "\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "rand:2:8": { "content": "\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "rand:3:8": { "content": "\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "rand:4:8": { "content": "\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null }, "rand:5:8": { "content": "\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/llamacpp-coder-bare-function-parallel", "family": "qwen3-xml", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected None, got '\\n\\n1\\n\\n\\n\\n'; tool_calls: expected ['special_function', 'special_function_with_opt'], got ['special_function_with_opt']" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream '\\n\\n1\\n\\n\\n\\n', got '\\n\\n\\n\\n'; tool_calls: nonstream ['special_function_with_opt'], got ['special_function', 'special_function_with_opt']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream '\\n\\n1\\n\\n\\n\\n', got '\\n\\n\\n\\n'; tool_calls: nonstream ['special_function_with_opt'], got ['special_function', 'special_function_with_opt']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream '\\n\\n1\\n\\n\\n\\n', got '\\n\\n\\n\\n'; tool_calls: nonstream ['special_function_with_opt'], got ['special_function', 'special_function_with_opt']" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream '\\n\\n1\\n\\n\\n\\n', got '\\n\\n\\n\\n'; tool_calls: nonstream ['special_function_with_opt'], got ['special_function', 'special_function_with_opt']" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream '\\n\\n1\\n\\n\\n\\n', got '\\n\\n\\n\\n'; tool_calls: nonstream ['special_function_with_opt'], got ['special_function', 'special_function_with_opt']" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream '\\n\\n1\\n\\n\\n\\n', got '\\n\\n\\n\\n'; tool_calls: nonstream ['special_function_with_opt'], got ['special_function', 'special_function_with_opt']" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream '\\n\\n1\\n\\n\\n\\n', got '\\n\\n\\n\\n'; tool_calls: nonstream ['special_function_with_opt'], got ['special_function', 'special_function_with_opt']" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream '\\n\\n1\\n\\n\\n\\n', got '\\n\\n\\n\\n'; tool_calls: nonstream ['special_function_with_opt'], got ['special_function', 'special_function_with_opt']" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "fail", "detail": "content contains ''; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "fail", "detail": "expected 2 calls ['special_function', 'special_function_with_opt'], got 1 ['special_function_with_opt']" }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": null, "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": null }, "model": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "tokenizer": { "repo": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3MoeForCausalLM" ], "model_type": "qwen3_moe" }, "hf_config_error": null, "chat_template_sha256": "5a38bfa05833266240066aedc497decc9b00cc0d3e3b8cceea98cf530196ab06", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|im_end|>", "id": 151645, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "\n\n1\n\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\": 1, \"arg2\": 2}" } ], "exception": null }, "streams": { "one": { "content": "\n\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\": 1, \"arg2\": 2}" } ], "exception": null }, "special": { "content": "\n\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\": 1, \"arg2\": 2}" } ], "exception": null }, "token": { "content": "\n\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\": 1, \"arg2\": 2}" } ], "exception": null }, "rand:1:8": { "content": "\n\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\": 1, \"arg2\": 2}" } ], "exception": null }, "rand:2:8": { "content": "\n\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\": 1, \"arg2\": 2}" } ], "exception": null }, "rand:3:8": { "content": "\n\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\": 1, \"arg2\": 2}" } ], "exception": null }, "rand:4:8": { "content": "\n\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\": 1, \"arg2\": 2}" } ], "exception": null }, "rand:5:8": { "content": "\n\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "special_function", "arguments_raw": "{\"arg1\": 1}" }, { "name": "special_function_with_opt", "arguments_raw": "{\"arg1\": 1, \"arg2\": 2}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/llamacpp-coder-unicode-cjk", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": null, "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": null }, "model": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "tokenizer": { "repo": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3MoeForCausalLM" ], "model_type": "qwen3_moe" }, "hf_config_error": null, "chat_template_sha256": "5a38bfa05833266240066aedc497decc9b00cc0d3e3b8cceea98cf530196ab06", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|im_end|>", "id": 151645, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "qwen3-xml/llamacpp-coder-html-value", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": null, "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": null }, "model": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "tokenizer": { "repo": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3MoeForCausalLM" ], "model_type": "qwen3_moe" }, "hf_config_error": null, "chat_template_sha256": "5a38bfa05833266240066aedc497decc9b00cc0d3e3b8cceea98cf530196ab06", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|im_end|>", "id": 151645, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "qwen3-xml/llamacpp-coder-array-of-objects", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": null, "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": null }, "model": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "tokenizer": { "repo": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3MoeForCausalLM" ], "model_type": "qwen3_moe" }, "hf_config_error": null, "chat_template_sha256": "5a38bfa05833266240066aedc497decc9b00cc0d3e3b8cceea98cf530196ab06", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|im_end|>", "id": 151645, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "qwen3-xml/vllm-coder-text-before-call-no-newline", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": null, "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": null }, "model": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "tokenizer": { "repo": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3MoeForCausalLM" ], "model_type": "qwen3_moe" }, "hf_config_error": null, "chat_template_sha256": "5a38bfa05833266240066aedc497decc9b00cc0d3e3b8cceea98cf530196ab06", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|im_end|>", "id": 151645, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "qwen3-xml/vllm-coder-anyof-coercion", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": null, "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": null }, "model": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "tokenizer": { "repo": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3MoeForCausalLM" ], "model_type": "qwen3_moe" }, "hf_config_error": null, "chat_template_sha256": "5a38bfa05833266240066aedc497decc9b00cc0d3e3b8cceea98cf530196ab06", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|im_end|>", "id": 151645, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "qwen3-xml/vllm-coder-missing-close-parameter", "family": "qwen3-xml", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you:', got 'Let me check the weather for you:\\n'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you:', got 'Let me check the weather for you:\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you:', got 'Let me check the weather for you:\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you:', got 'Let me check the weather for you:\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you:', got 'Let me check the weather for you:\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you:', got 'Let me check the weather for you:\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you:', got 'Let me check the weather for you:\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you:', got 'Let me check the weather for you:\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you:', got 'Let me check the weather for you:\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": null, "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": null }, "model": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "tokenizer": { "repo": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3MoeForCausalLM" ], "model_type": "qwen3_moe" }, "hf_config_error": null, "chat_template_sha256": "5a38bfa05833266240066aedc497decc9b00cc0d3e3b8cceea98cf530196ab06", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 2, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|im_end|>", "id": 151645, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "Let me check the weather for you:\n", "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Dallas\", \"state\": \"TX\", \"unit\": \"fahrenheit\"}" } ], "exception": null }, "streams": { "one": { "content": "Let me check the weather for you:\n", "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Dallas\", \"state\": \"TX\", \"unit\": \"fahrenheit\"}" } ], "exception": null }, "special": { "content": "Let me check the weather for you:\n", "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Dallas\", \"state\": \"TX\", \"unit\": \"fahrenheit\"}" } ], "exception": null }, "token": { "content": "Let me check the weather for you:\n", "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Dallas\", \"state\": \"TX\", \"unit\": \"fahrenheit\"}" } ], "exception": null }, "rand:1:8": { "content": "Let me check the weather for you:\n", "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Dallas\", \"state\": \"TX\", \"unit\": \"fahrenheit\"}" } ], "exception": null }, "rand:2:8": { "content": "Let me check the weather for you:\n", "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Dallas\", \"state\": \"TX\", \"unit\": \"fahrenheit\"}" } ], "exception": null }, "rand:3:8": { "content": "Let me check the weather for you:\n", "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Dallas\", \"state\": \"TX\", \"unit\": \"fahrenheit\"}" } ], "exception": null }, "rand:4:8": { "content": "Let me check the weather for you:\n", "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Dallas\", \"state\": \"TX\", \"unit\": \"fahrenheit\"}" } ], "exception": null }, "rand:5:8": { "content": "Let me check the weather for you:\n", "reasoning_content": null, "tool_calls": [ { "name": "get_current_weather", "arguments_raw": "{\"city\": \"Dallas\", \"state\": \"TX\", \"unit\": \"fahrenheit\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/bug-missing-close-parameter-before-function", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.8-27B", "tokenizer": { "repo": "Qwen/Qwen3.8-27B", "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 248068, 271, 248069, 271 ] }, "stop": { "token": "<|im_end|>", "id": 248046, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "qwen3-xml/bug-coder-missing-tool-call-opener", "family": "qwen3-xml", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected 'I will first check for all Markdown files in the current directory and its subdirectories, and then list them using the glob tool.', got 'I will first check for all Markdown files in the current directory and its subdirectories, and then list them using the glob tool.\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n'; content contains ''; content contains ''" }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": null, "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": null }, "model": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "tokenizer": { "repo": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3MoeForCausalLM" ], "model_type": "qwen3_moe" }, "hf_config_error": null, "chat_template_sha256": "5a38bfa05833266240066aedc497decc9b00cc0d3e3b8cceea98cf530196ab06", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|im_end|>", "id": 151645, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "I will first check for all Markdown files in the current directory and its subdirectories, and then list them using the glob tool.\n\n\n\n**/*.md\n\n\n", "reasoning_content": null, "tool_calls": [], "exception": null }, "streams": { "one": { "content": "I will first check for all Markdown files in the current directory and its subdirectories, and then list them using the glob tool.\n\n\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "glob", "arguments_raw": "{\"pattern\": \"**/*.md\"}" } ], "exception": null }, "special": { "content": "I will first check for all Markdown files in the current directory and its subdirectories, and then list them using the glob tool.\n\n\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "glob", "arguments_raw": "{\"pattern\": \"**/*.md\"}" } ], "exception": null }, "token": { "content": "I will first check for all Markdown files in the current directory and its subdirectories, and then list them using the glob tool.\n\n\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "glob", "arguments_raw": "{\"pattern\": \"**/*.md\"}" } ], "exception": null }, "rand:1:8": { "content": "I will first check for all Markdown files in the current directory and its subdirectories, and then list them using the glob tool.\n\n\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "glob", "arguments_raw": "{\"pattern\": \"**/*.md\"}" } ], "exception": null }, "rand:2:8": { "content": "I will first check for all Markdown files in the current directory and its subdirectories, and then list them using the glob tool.\n\n\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "glob", "arguments_raw": "{\"pattern\": \"**/*.md\"}" } ], "exception": null }, "rand:3:8": { "content": "I will first check for all Markdown files in the current directory and its subdirectories, and then list them using the glob tool.\n\n\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "glob", "arguments_raw": "{\"pattern\": \"**/*.md\"}" } ], "exception": null }, "rand:4:8": { "content": "I will first check for all Markdown files in the current directory and its subdirectories, and then list them using the glob tool.\n\n\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "glob", "arguments_raw": "{\"pattern\": \"**/*.md\"}" } ], "exception": null }, "rand:5:8": { "content": "I will first check for all Markdown files in the current directory and its subdirectories, and then list them using the glob tool.\n\n\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "glob", "arguments_raw": "{\"pattern\": \"**/*.md\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/bug-coder-text-after-call", "family": "qwen3-xml", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected 'beforeafter', got 'before'" }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream 'before', got 'beforeafter'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream 'before', got 'beforeafter'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream 'before', got 'beforeafter'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream 'before', got 'beforeafter'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream 'before', got 'beforeafter'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream 'before', got 'beforeafter'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream 'before', got 'beforeafter'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream 'before', got 'beforeafter'" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": null, "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": null }, "model": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "tokenizer": { "repo": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3MoeForCausalLM" ], "model_type": "qwen3_moe" }, "hf_config_error": null, "chat_template_sha256": "5a38bfa05833266240066aedc497decc9b00cc0d3e3b8cceea98cf530196ab06", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|im_end|>", "id": 151645, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "before", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "streams": { "one": { "content": "beforeafter", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "special": { "content": "beforeafter", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "token": { "content": "beforeafter", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "rand:1:8": { "content": "beforeafter", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "rand:2:8": { "content": "beforeafter", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "rand:3:8": { "content": "beforeafter", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "rand:4:8": { "content": "beforeafter", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "rand:5:8": { "content": "beforeafter", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/bug-coder-number-outside-int64", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": null, "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": null }, "model": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "tokenizer": { "repo": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3MoeForCausalLM" ], "model_type": "qwen3_moe" }, "hf_config_error": null, "chat_template_sha256": "5a38bfa05833266240066aedc497decc9b00cc0d3e3b8cceea98cf530196ab06", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 1, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|im_end|>", "id": 151645, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "qwen3-xml/q38-single-call", "family": "qwen3-xml", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'The user wants the weather in Paris. I should call get_weather.', got 'The user wants the weather in Paris. I should call get_weather.\\n'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'The user wants the weather in Paris. I should call get_weather.', got 'The user wants the weather in Paris. I should call get_weather.\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'The user wants the weather in Paris. I should call get_weather.', got 'The user wants the weather in Paris. I should call get_weather.\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'The user wants the weather in Paris. I should call get_weather.', got 'The user wants the weather in Paris. I should call get_weather.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'The user wants the weather in Paris. I should call get_weather.', got 'The user wants the weather in Paris. I should call get_weather.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'The user wants the weather in Paris. I should call get_weather.', got 'The user wants the weather in Paris. I should call get_weather.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'The user wants the weather in Paris. I should call get_weather.', got 'The user wants the weather in Paris. I should call get_weather.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'The user wants the weather in Paris. I should call get_weather.', got 'The user wants the weather in Paris. I should call get_weather.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'The user wants the weather in Paris. I should call get_weather.', got 'The user wants the weather in Paris. I should call get_weather.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.8-27B", "tokenizer": { "repo": "Qwen/Qwen3.8-27B", "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 248045, 74455, 198, 248068, 198 ] }, "stop": { "token": "<|im_end|>", "id": 248046, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "\n\n", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "streams": { "one": { "content": "\n\n", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "special": { "content": "\n\n", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "token": { "content": "\n\n", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:1:8": { "content": "\n\n", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:3:8": { "content": "\n\n", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:4:8": { "content": "\n\n", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:5:8": { "content": "\n\n", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/q38-parallel-calls", "family": "qwen3-xml", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'; reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'; reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'; reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'; reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'; reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'; reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'; reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n'; reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): nonstream '\\n\\n', got '\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): nonstream '\\n\\n', got '\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream '\\n\\n', got '\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream '\\n\\n', got '\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream '\\n\\n', got '\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream '\\n\\n', got '\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream '\\n\\n', got '\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream '\\n\\n', got '\\n\\n\\n'" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.8-27B", "tokenizer": { "repo": "Qwen/Qwen3.8-27B", "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 248045, 74455, 198, 248068, 198 ] }, "stop": { "token": "<|im_end|>", "id": 248046, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "\n\n", "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "streams": { "one": { "content": "\n\n\n", "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "special": { "content": "\n\n\n", "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "token": { "content": "\n\n\n", "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:1:8": { "content": "\n\n\n", "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:2:8": { "content": "\n\n\n", "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:3:8": { "content": "\n\n\n", "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:4:8": { "content": "\n\n\n", "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:5:8": { "content": "\n\n\n", "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/q38-three-parallel-calls", "family": "qwen3-xml", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Three cities, three calls.', got 'Three cities, three calls.\\n'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n\\n'; reasoning_content (whitespace only): expected 'Three cities, three calls.', got 'Three cities, three calls.\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n\\n'; reasoning_content (whitespace only): expected 'Three cities, three calls.', got 'Three cities, three calls.\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n\\n'; reasoning_content (whitespace only): expected 'Three cities, three calls.', got 'Three cities, three calls.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n\\n'; reasoning_content (whitespace only): expected 'Three cities, three calls.', got 'Three cities, three calls.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n\\n'; reasoning_content (whitespace only): expected 'Three cities, three calls.', got 'Three cities, three calls.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n\\n'; reasoning_content (whitespace only): expected 'Three cities, three calls.', got 'Three cities, three calls.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n\\n'; reasoning_content (whitespace only): expected 'Three cities, three calls.', got 'Three cities, three calls.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n\\n\\n'; reasoning_content (whitespace only): expected 'Three cities, three calls.', got 'Three cities, three calls.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): nonstream '\\n\\n', got '\\n\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): nonstream '\\n\\n', got '\\n\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream '\\n\\n', got '\\n\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream '\\n\\n', got '\\n\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream '\\n\\n', got '\\n\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream '\\n\\n', got '\\n\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream '\\n\\n', got '\\n\\n\\n\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream '\\n\\n', got '\\n\\n\\n\\n'" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.8-27B", "tokenizer": { "repo": "Qwen/Qwen3.8-27B", "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 248045, 74455, 198, 248068, 198 ] }, "stop": { "token": "<|im_end|>", "id": 248046, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "\n\n", "reasoning_content": "Three cities, three calls.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"f\"}" } ], "exception": null }, "streams": { "one": { "content": "\n\n\n\n", "reasoning_content": "Three cities, three calls.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"f\"}" } ], "exception": null }, "special": { "content": "\n\n\n\n", "reasoning_content": "Three cities, three calls.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"f\"}" } ], "exception": null }, "token": { "content": "\n\n\n\n", "reasoning_content": "Three cities, three calls.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"f\"}" } ], "exception": null }, "rand:1:8": { "content": "\n\n\n\n", "reasoning_content": "Three cities, three calls.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"f\"}" } ], "exception": null }, "rand:2:8": { "content": "\n\n\n\n", "reasoning_content": "Three cities, three calls.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"f\"}" } ], "exception": null }, "rand:3:8": { "content": "\n\n\n\n", "reasoning_content": "Three cities, three calls.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"f\"}" } ], "exception": null }, "rand:4:8": { "content": "\n\n\n\n", "reasoning_content": "Three cities, three calls.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"f\"}" } ], "exception": null }, "rand:5:8": { "content": "\n\n\n\n", "reasoning_content": "Three cities, three calls.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Berlin\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Tokyo\", \"unit\": \"f\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/q38-unicode-emoji", "family": "qwen3-xml", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected '用户想知道东京的天气。🌦️', got '用户想知道东京的天气。🌦️\\n'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected '用户想知道东京的天气。🌦️', got '用户想知道东京的天气。🌦️\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected '用户想知道东京的天气。🌦️', got '用户想知道东京的天气。🌦️\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected '用户想知道东京的天气。🌦️', got '用户想知道东京的天气。🌦️\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected '用户想知道东京的天气。🌦️', got '用户想知道东京的天气。🌦️\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected '用户想知道东京的天气。🌦️', got '用户想知道东京的天气。🌦️\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected '用户想知道东京的天气。🌦️', got '用户想知道东京的天气。🌦️\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected '用户想知道东京的天气。🌦️', got '用户想知道东京的天气。🌦️\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected '用户想知道东京的天气。🌦️', got '用户想知道东京的天气。🌦️\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.8-27B", "tokenizer": { "repo": "Qwen/Qwen3.8-27B", "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 248045, 74455, 198, 248068, 198 ] }, "stop": { "token": "<|im_end|>", "id": 248046, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "\n\n", "reasoning_content": "用户想知道东京的天气。🌦️\n", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"東京の天気 ☀️🌧️ — «prévisions» für Zürich 🇨🇭\"}" } ], "exception": null }, "streams": { "one": { "content": "\n\n", "reasoning_content": "用户想知道东京的天气。🌦️\n", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"東京の天気 ☀️🌧️ — «prévisions» für Zürich 🇨🇭\"}" } ], "exception": null }, "special": { "content": "\n\n", "reasoning_content": "用户想知道东京的天气。🌦️\n", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"東京の天気 ☀️🌧️ — «prévisions» für Zürich 🇨🇭\"}" } ], "exception": null }, "token": { "content": "\n\n", "reasoning_content": "用户想知道东京的天气。🌦️\n", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"東京の天気 ☀️🌧️ — «prévisions» für Zürich 🇨🇭\"}" } ], "exception": null }, "rand:1:8": { "content": "\n\n", "reasoning_content": "用户想知道东京的天气。🌦️\n", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"東京の天気 ☀️🌧️ — «prévisions» für Zürich 🇨🇭\"}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": "用户想知道东京的天气。🌦️\n", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"東京の天気 ☀️🌧️ — «prévisions» für Zürich 🇨🇭\"}" } ], "exception": null }, "rand:3:8": { "content": "\n\n", "reasoning_content": "用户想知道东京的天气。🌦️\n", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"東京の天気 ☀️🌧️ — «prévisions» für Zürich 🇨🇭\"}" } ], "exception": null }, "rand:4:8": { "content": "\n\n", "reasoning_content": "用户想知道东京的天气。🌦️\n", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"東京の天気 ☀️🌧️ — «prévisions» für Zürich 🇨🇭\"}" } ], "exception": null }, "rand:5:8": { "content": "\n\n", "reasoning_content": "用户想知道东京的天气。🌦️\n", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"東京の天気 ☀️🌧️ — «prévisions» für Zürich 🇨🇭\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/q38-nested-json", "family": "qwen3-xml", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Search with structured filters.', got 'Search with structured filters.\\n'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Search with structured filters.', got 'Search with structured filters.\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Search with structured filters.', got 'Search with structured filters.\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Search with structured filters.', got 'Search with structured filters.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Search with structured filters.', got 'Search with structured filters.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Search with structured filters.', got 'Search with structured filters.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Search with structured filters.', got 'Search with structured filters.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Search with structured filters.', got 'Search with structured filters.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Search with structured filters.', got 'Search with structured filters.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.8-27B", "tokenizer": { "repo": "Qwen/Qwen3.8-27B", "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 248045, 74455, 198, 248068, 198 ] }, "stop": { "token": "<|im_end|>", "id": 248046, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "\n\n", "reasoning_content": "Search with structured filters.\n", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"hotels\", \"filters\": {\"price\": {\"min\": 50, \"max\": 120.5}, \"tags\": [\"pool\", \"wifi\"], \"open\": null}, \"sites\": [\"a.example\", \"b.example\"]}" } ], "exception": null }, "streams": { "one": { "content": "\n\n", "reasoning_content": "Search with structured filters.\n", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"hotels\", \"filters\": {\"price\": {\"min\": 50, \"max\": 120.5}, \"tags\": [\"pool\", \"wifi\"], \"open\": null}, \"sites\": [\"a.example\", \"b.example\"]}" } ], "exception": null }, "special": { "content": "\n\n", "reasoning_content": "Search with structured filters.\n", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"hotels\", \"filters\": {\"price\": {\"min\": 50, \"max\": 120.5}, \"tags\": [\"pool\", \"wifi\"], \"open\": null}, \"sites\": [\"a.example\", \"b.example\"]}" } ], "exception": null }, "token": { "content": "\n\n", "reasoning_content": "Search with structured filters.\n", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"hotels\", \"filters\": {\"price\": {\"min\": 50, \"max\": 120.5}, \"tags\": [\"pool\", \"wifi\"], \"open\": null}, \"sites\": [\"a.example\", \"b.example\"]}" } ], "exception": null }, "rand:1:8": { "content": "\n\n", "reasoning_content": "Search with structured filters.\n", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"hotels\", \"filters\": {\"price\": {\"min\": 50, \"max\": 120.5}, \"tags\": [\"pool\", \"wifi\"], \"open\": null}, \"sites\": [\"a.example\", \"b.example\"]}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": "Search with structured filters.\n", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"hotels\", \"filters\": {\"price\": {\"min\": 50, \"max\": 120.5}, \"tags\": [\"pool\", \"wifi\"], \"open\": null}, \"sites\": [\"a.example\", \"b.example\"]}" } ], "exception": null }, "rand:3:8": { "content": "\n\n", "reasoning_content": "Search with structured filters.\n", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"hotels\", \"filters\": {\"price\": {\"min\": 50, \"max\": 120.5}, \"tags\": [\"pool\", \"wifi\"], \"open\": null}, \"sites\": [\"a.example\", \"b.example\"]}" } ], "exception": null }, "rand:4:8": { "content": "\n\n", "reasoning_content": "Search with structured filters.\n", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"hotels\", \"filters\": {\"price\": {\"min\": 50, \"max\": 120.5}, \"tags\": [\"pool\", \"wifi\"], \"open\": null}, \"sites\": [\"a.example\", \"b.example\"]}" } ], "exception": null }, "rand:5:8": { "content": "\n\n", "reasoning_content": "Search with structured filters.\n", "tool_calls": [ { "name": "search", "arguments_raw": "{\"query\": \"hotels\", \"filters\": {\"price\": {\"min\": 50, \"max\": 120.5}, \"tags\": [\"pool\", \"wifi\"], \"open\": null}, \"sites\": [\"a.example\", \"b.example\"]}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/q38-empty-arguments", "family": "qwen3-xml", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Get the time.', got 'Get the time.\\n'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Get the time.', got 'Get the time.\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Get the time.', got 'Get the time.\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Get the time.', got 'Get the time.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Get the time.', got 'Get the time.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Get the time.', got 'Get the time.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Get the time.', got 'Get the time.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Get the time.', got 'Get the time.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Get the time.', got 'Get the time.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.8-27B", "tokenizer": { "repo": "Qwen/Qwen3.8-27B", "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 248045, 74455, 198, 248068, 198 ] }, "stop": { "token": "<|im_end|>", "id": 248046, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "\n\n", "reasoning_content": "Get the time.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "streams": { "one": { "content": "\n\n", "reasoning_content": "Get the time.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": "\n\n", "reasoning_content": "Get the time.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": "\n\n", "reasoning_content": "Get the time.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:1:8": { "content": "\n\n", "reasoning_content": "Get the time.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": "Get the time.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:3:8": { "content": "\n\n", "reasoning_content": "Get the time.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:4:8": { "content": "\n\n", "reasoning_content": "Get the time.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:5:8": { "content": "\n\n", "reasoning_content": "Get the time.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/q38-marker-in-arguments", "family": "qwen3-xml", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write the note.', got 'Write the note.\\n'; tool_calls[0].arguments: expected '{\"path\": \"notes.md\", \"content\": \"Close tags inline: and and are text here.\"}', got '{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write the note.', got 'Write the note.\\n'; tool_calls[0].arguments: expected '{\"path\": \"notes.md\", \"content\": \"Close tags inline: and and are text here.\"}', got '{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}, \"path\": \" are text here.\"}'" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write the note.', got 'Write the note.\\n'; tool_calls[0].arguments: expected '{\"path\": \"notes.md\", \"content\": \"Close tags inline: and and are text here.\"}', got '{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}, \"path\": \" are text here.\"}'" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write the note.', got 'Write the note.\\n'; tool_calls[0].arguments: expected '{\"path\": \"notes.md\", \"content\": \"Close tags inline: and and are text here.\"}', got '{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}, \"path\": \" are text here.\"}'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write the note.', got 'Write the note.\\n'; tool_calls[0].arguments: expected '{\"path\": \"notes.md\", \"content\": \"Close tags inline: and and are text here.\"}', got '{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}, \"path\": \" are text here.\"}'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write the note.', got 'Write the note.\\n'; tool_calls[0].arguments: expected '{\"path\": \"notes.md\", \"content\": \"Close tags inline: and and are text here.\"}', got '{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}, \"path\": \" are text here.\"}'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write the note.', got 'Write the note.\\n'; tool_calls[0].arguments: expected '{\"path\": \"notes.md\", \"content\": \"Close tags inline: and and are text here.\"}', got '{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}, \"path\": \" are text here.\"}'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write the note.', got 'Write the note.\\n'; tool_calls[0].arguments: expected '{\"path\": \"notes.md\", \"content\": \"Close tags inline: and and are text here.\"}', got '{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}, \"path\": \" are text here.\"}'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write the note.', got 'Write the note.\\n'; tool_calls[0].arguments: expected '{\"path\": \"notes.md\", \"content\": \"Close tags inline: and and are text here.\"}', got '{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}, \"path\": \" are text here.\"}'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}', got '{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}, \"path\": \" are text here.\"}'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}', got '{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}, \"path\": \" are text here.\"}'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}', got '{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}, \"path\": \" are text here.\"}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}', got '{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}, \"path\": \" are text here.\"}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}', got '{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}, \"path\": \" are text here.\"}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}', got '{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}, \"path\": \" are text here.\"}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}', got '{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}, \"path\": \" are text here.\"}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}', got '{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}, \"path\": \" are text here.\"}'" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}, \"path\": \" are text here.\"}': Extra data: line 1 column 55 (char 54))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}, \"path\": \" are text here.\"}': Extra data: line 1 column 55 (char 54))" }, { "check": "arguments_json", "strategy": "token", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}, \"path\": \" are text here.\"}': Extra data: line 1 column 55 (char 54))" }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}, \"path\": \" are text here.\"}': Extra data: line 1 column 55 (char 54))" }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}, \"path\": \" are text here.\"}': Extra data: line 1 column 55 (char 54))" }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}, \"path\": \" are text here.\"}': Extra data: line 1 column 55 (char 54))" }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}, \"path\": \" are text here.\"}': Extra data: line 1 column 55 (char 54))" }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}, \"path\": \" are text here.\"}': Extra data: line 1 column 55 (char 54))" }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.8-27B", "tokenizer": { "repo": "Qwen/Qwen3.8-27B", "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 248045, 74455, 198, 248068, 198 ] }, "stop": { "token": "<|im_end|>", "id": 248046, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "\n\n", "reasoning_content": "Write the note.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}" } ], "exception": null }, "streams": { "one": { "content": "\n\n", "reasoning_content": "Write the note.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}, \"path\": \" are text here.\"}" } ], "exception": null }, "special": { "content": "\n\n", "reasoning_content": "Write the note.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}, \"path\": \" are text here.\"}" } ], "exception": null }, "token": { "content": "\n\n", "reasoning_content": "Write the note.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}, \"path\": \" are text here.\"}" } ], "exception": null }, "rand:1:8": { "content": "\n\n", "reasoning_content": "Write the note.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}, \"path\": \" are text here.\"}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": "Write the note.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}, \"path\": \" are text here.\"}" } ], "exception": null }, "rand:3:8": { "content": "\n\n", "reasoning_content": "Write the note.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}, \"path\": \" are text here.\"}" } ], "exception": null }, "rand:4:8": { "content": "\n\n", "reasoning_content": "Write the note.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}, \"path\": \" are text here.\"}" } ], "exception": null }, "rand:5:8": { "content": "\n\n", "reasoning_content": "Write the note.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"notes.md\", \"content\": \"Close tags inline: \"}, \"path\": \" are text here.\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/q38-tool-call-token-in-arguments", "family": "qwen3-xml", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Document the format.', got 'Document the format.\\n'; tool_calls[0].arguments: expected '{\"path\": \"fmt.md\", \"content\": \"Qwen wraps calls in ... blocks.\"}', got '{\"path\": \"fmt.md\", \"content\": \"Qwen wraps calls in ... \"}'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Document the format.', got 'Document the format.\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Document the format.', got 'Document the format.\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Document the format.', got 'Document the format.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Document the format.', got 'Document the format.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Document the format.', got 'Document the format.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Document the format.', got 'Document the format.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Document the format.', got 'Document the format.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Document the format.', got 'Document the format.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"path\": \"fmt.md\", \"content\": \"Qwen wraps calls in ... \"}', got '{\"path\": \"fmt.md\", \"content\": \"Qwen wraps calls in ... blocks.\"}'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"path\": \"fmt.md\", \"content\": \"Qwen wraps calls in ... \"}', got '{\"path\": \"fmt.md\", \"content\": \"Qwen wraps calls in ... blocks.\"}'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"path\": \"fmt.md\", \"content\": \"Qwen wraps calls in ... \"}', got '{\"path\": \"fmt.md\", \"content\": \"Qwen wraps calls in ... blocks.\"}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"path\": \"fmt.md\", \"content\": \"Qwen wraps calls in ... \"}', got '{\"path\": \"fmt.md\", \"content\": \"Qwen wraps calls in ... blocks.\"}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"path\": \"fmt.md\", \"content\": \"Qwen wraps calls in ... \"}', got '{\"path\": \"fmt.md\", \"content\": \"Qwen wraps calls in ... blocks.\"}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"path\": \"fmt.md\", \"content\": \"Qwen wraps calls in ... \"}', got '{\"path\": \"fmt.md\", \"content\": \"Qwen wraps calls in ... blocks.\"}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"path\": \"fmt.md\", \"content\": \"Qwen wraps calls in ... \"}', got '{\"path\": \"fmt.md\", \"content\": \"Qwen wraps calls in ... blocks.\"}'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"path\": \"fmt.md\", \"content\": \"Qwen wraps calls in ... \"}', got '{\"path\": \"fmt.md\", \"content\": \"Qwen wraps calls in ... blocks.\"}'" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.8-27B", "tokenizer": { "repo": "Qwen/Qwen3.8-27B", "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 248045, 74455, 198, 248068, 198 ] }, "stop": { "token": "<|im_end|>", "id": 248046, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "\n\n", "reasoning_content": "Document the format.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"fmt.md\", \"content\": \"Qwen wraps calls in ... \"}" } ], "exception": null }, "streams": { "one": { "content": "\n\n", "reasoning_content": "Document the format.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"fmt.md\", \"content\": \"Qwen wraps calls in ... blocks.\"}" } ], "exception": null }, "special": { "content": "\n\n", "reasoning_content": "Document the format.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"fmt.md\", \"content\": \"Qwen wraps calls in ... blocks.\"}" } ], "exception": null }, "token": { "content": "\n\n", "reasoning_content": "Document the format.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"fmt.md\", \"content\": \"Qwen wraps calls in ... blocks.\"}" } ], "exception": null }, "rand:1:8": { "content": "\n\n", "reasoning_content": "Document the format.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"fmt.md\", \"content\": \"Qwen wraps calls in ... blocks.\"}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": "Document the format.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"fmt.md\", \"content\": \"Qwen wraps calls in ... blocks.\"}" } ], "exception": null }, "rand:3:8": { "content": "\n\n", "reasoning_content": "Document the format.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"fmt.md\", \"content\": \"Qwen wraps calls in ... blocks.\"}" } ], "exception": null }, "rand:4:8": { "content": "\n\n", "reasoning_content": "Document the format.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"fmt.md\", \"content\": \"Qwen wraps calls in ... blocks.\"}" } ], "exception": null }, "rand:5:8": { "content": "\n\n", "reasoning_content": "Document the format.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"fmt.md\", \"content\": \"Qwen wraps calls in ... blocks.\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/q38-text-before-call", "family": "qwen3-xml", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you.', got '\\n\\nLet me check the weather for you.\\n\\n'; reasoning_content (whitespace only): expected 'The user wants the weather in Paris. I should call get_weather.', got 'The user wants the weather in Paris. I should call get_weather.\\n'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you.', got '\\n\\nLet me check the weather for you.\\n\\n'; reasoning_content (whitespace only): expected 'The user wants the weather in Paris. I should call get_weather.', got 'The user wants the weather in Paris. I should call get_weather.\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you.', got '\\n\\nLet me check the weather for you.\\n\\n'; reasoning_content (whitespace only): expected 'The user wants the weather in Paris. I should call get_weather.', got 'The user wants the weather in Paris. I should call get_weather.\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you.', got '\\n\\nLet me check the weather for you.\\n\\n'; reasoning_content (whitespace only): expected 'The user wants the weather in Paris. I should call get_weather.', got 'The user wants the weather in Paris. I should call get_weather.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you.', got '\\n\\nLet me check the weather for you.\\n\\n'; reasoning_content (whitespace only): expected 'The user wants the weather in Paris. I should call get_weather.', got 'The user wants the weather in Paris. I should call get_weather.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you.', got '\\n\\nLet me check the weather for you.\\n\\n'; reasoning_content (whitespace only): expected 'The user wants the weather in Paris. I should call get_weather.', got 'The user wants the weather in Paris. I should call get_weather.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you.', got '\\n\\nLet me check the weather for you.\\n\\n'; reasoning_content (whitespace only): expected 'The user wants the weather in Paris. I should call get_weather.', got 'The user wants the weather in Paris. I should call get_weather.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you.', got '\\n\\nLet me check the weather for you.\\n\\n'; reasoning_content (whitespace only): expected 'The user wants the weather in Paris. I should call get_weather.', got 'The user wants the weather in Paris. I should call get_weather.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you.', got '\\n\\nLet me check the weather for you.\\n\\n'; reasoning_content (whitespace only): expected 'The user wants the weather in Paris. I should call get_weather.', got 'The user wants the weather in Paris. I should call get_weather.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.8-27B", "tokenizer": { "repo": "Qwen/Qwen3.8-27B", "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 248045, 74455, 198, 248068, 198 ] }, "stop": { "token": "<|im_end|>", "id": 248046, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "\n\nLet me check the weather for you.\n\n", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "streams": { "one": { "content": "\n\nLet me check the weather for you.\n\n", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "special": { "content": "\n\nLet me check the weather for you.\n\n", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "token": { "content": "\n\nLet me check the weather for you.\n\n", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "rand:1:8": { "content": "\n\nLet me check the weather for you.\n\n", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "rand:2:8": { "content": "\n\nLet me check the weather for you.\n\n", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "rand:3:8": { "content": "\n\nLet me check the weather for you.\n\n", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "rand:4:8": { "content": "\n\nLet me check the weather for you.\n\n", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null }, "rand:5:8": { "content": "\n\nLet me check the weather for you.\n\n", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/q38-no-call", "family": "qwen3-xml", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "content (whitespace only): expected 'Hello! How can I help you today?', got '\\n\\nHello! How can I help you today?'; reasoning_content (whitespace only): expected 'A simple greeting; no tool is needed.', got 'A simple greeting; no tool is needed.\\n'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected 'Hello! How can I help you today?', got '\\n\\nHello! How can I help you today?'; reasoning_content (whitespace only): expected 'A simple greeting; no tool is needed.', got 'A simple greeting; no tool is needed.\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected 'Hello! How can I help you today?', got '\\n\\nHello! How can I help you today?'; reasoning_content (whitespace only): expected 'A simple greeting; no tool is needed.', got 'A simple greeting; no tool is needed.\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected 'Hello! How can I help you today?', got '\\n\\nHello! How can I help you today?'; reasoning_content (whitespace only): expected 'A simple greeting; no tool is needed.', got 'A simple greeting; no tool is needed.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Hello! How can I help you today?', got '\\n\\nHello! How can I help you today?'; reasoning_content (whitespace only): expected 'A simple greeting; no tool is needed.', got 'A simple greeting; no tool is needed.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Hello! How can I help you today?', got '\\n\\nHello! How can I help you today?'; reasoning_content (whitespace only): expected 'A simple greeting; no tool is needed.', got 'A simple greeting; no tool is needed.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Hello! How can I help you today?', got '\\n\\nHello! How can I help you today?'; reasoning_content (whitespace only): expected 'A simple greeting; no tool is needed.', got 'A simple greeting; no tool is needed.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Hello! How can I help you today?', got '\\n\\nHello! How can I help you today?'; reasoning_content (whitespace only): expected 'A simple greeting; no tool is needed.', got 'A simple greeting; no tool is needed.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Hello! How can I help you today?', got '\\n\\nHello! How can I help you today?'; reasoning_content (whitespace only): expected 'A simple greeting; no tool is needed.', got 'A simple greeting; no tool is needed.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.8-27B", "tokenizer": { "repo": "Qwen/Qwen3.8-27B", "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 248045, 74455, 198, 248068, 198 ] }, "stop": { "token": "<|im_end|>", "id": 248046, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "\n\nHello! How can I help you today?", "reasoning_content": "A simple greeting; no tool is needed.\n", "tool_calls": [], "exception": null }, "streams": { "one": { "content": "\n\nHello! How can I help you today?", "reasoning_content": "A simple greeting; no tool is needed.\n", "tool_calls": [], "exception": null }, "special": { "content": "\n\nHello! How can I help you today?", "reasoning_content": "A simple greeting; no tool is needed.\n", "tool_calls": [], "exception": null }, "token": { "content": "\n\nHello! How can I help you today?", "reasoning_content": "A simple greeting; no tool is needed.\n", "tool_calls": [], "exception": null }, "rand:1:8": { "content": "\n\nHello! How can I help you today?", "reasoning_content": "A simple greeting; no tool is needed.\n", "tool_calls": [], "exception": null }, "rand:2:8": { "content": "\n\nHello! How can I help you today?", "reasoning_content": "A simple greeting; no tool is needed.\n", "tool_calls": [], "exception": null }, "rand:3:8": { "content": "\n\nHello! How can I help you today?", "reasoning_content": "A simple greeting; no tool is needed.\n", "tool_calls": [], "exception": null }, "rand:4:8": { "content": "\n\nHello! How can I help you today?", "reasoning_content": "A simple greeting; no tool is needed.\n", "tool_calls": [], "exception": null }, "rand:5:8": { "content": "\n\nHello! How can I help you today?", "reasoning_content": "A simple greeting; no tool is needed.\n", "tool_calls": [], "exception": null } } } }, { "fixture_id": "qwen3-xml/q38-multiline-string", "family": "qwen3-xml", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write the script.', got 'Write the script.\\n'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write the script.', got 'Write the script.\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write the script.', got 'Write the script.\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write the script.', got 'Write the script.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write the script.', got 'Write the script.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write the script.', got 'Write the script.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write the script.', got 'Write the script.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write the script.', got 'Write the script.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write the script.', got 'Write the script.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.8-27B", "tokenizer": { "repo": "Qwen/Qwen3.8-27B", "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 248045, 74455, 198, 248068, 198 ] }, "stop": { "token": "<|im_end|>", "id": 248046, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "\n\n", "reasoning_content": "Write the script.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"greet.py\", \"content\": \"def greet(name: str) -> str:\\n \\\"\\\"\\\"Say hi.\\\"\\\"\\\"\\n return f\\\"Hello, {name}!\\\"\\n\\n\\nprint(greet(\\\"Zoë\\\"))\"}" } ], "exception": null }, "streams": { "one": { "content": "\n\n", "reasoning_content": "Write the script.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"greet.py\", \"content\": \"def greet(name: str) -> str:\\n \\\"\\\"\\\"Say hi.\\\"\\\"\\\"\\n return f\\\"Hello, {name}!\\\"\\n\\n\\nprint(greet(\\\"Zoë\\\"))\"}" } ], "exception": null }, "special": { "content": "\n\n", "reasoning_content": "Write the script.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"greet.py\", \"content\": \"def greet(name: str) -> str:\\n \\\"\\\"\\\"Say hi.\\\"\\\"\\\"\\n return f\\\"Hello, {name}!\\\"\\n\\n\\nprint(greet(\\\"Zoë\\\"))\"}" } ], "exception": null }, "token": { "content": "\n\n", "reasoning_content": "Write the script.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"greet.py\", \"content\": \"def greet(name: str) -> str:\\n \\\"\\\"\\\"Say hi.\\\"\\\"\\\"\\n return f\\\"Hello, {name}!\\\"\\n\\n\\nprint(greet(\\\"Zoë\\\"))\"}" } ], "exception": null }, "rand:1:8": { "content": "\n\n", "reasoning_content": "Write the script.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"greet.py\", \"content\": \"def greet(name: str) -> str:\\n \\\"\\\"\\\"Say hi.\\\"\\\"\\\"\\n return f\\\"Hello, {name}!\\\"\\n\\n\\nprint(greet(\\\"Zoë\\\"))\"}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": "Write the script.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"greet.py\", \"content\": \"def greet(name: str) -> str:\\n \\\"\\\"\\\"Say hi.\\\"\\\"\\\"\\n return f\\\"Hello, {name}!\\\"\\n\\n\\nprint(greet(\\\"Zoë\\\"))\"}" } ], "exception": null }, "rand:3:8": { "content": "\n\n", "reasoning_content": "Write the script.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"greet.py\", \"content\": \"def greet(name: str) -> str:\\n \\\"\\\"\\\"Say hi.\\\"\\\"\\\"\\n return f\\\"Hello, {name}!\\\"\\n\\n\\nprint(greet(\\\"Zoë\\\"))\"}" } ], "exception": null }, "rand:4:8": { "content": "\n\n", "reasoning_content": "Write the script.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"greet.py\", \"content\": \"def greet(name: str) -> str:\\n \\\"\\\"\\\"Say hi.\\\"\\\"\\\"\\n return f\\\"Hello, {name}!\\\"\\n\\n\\nprint(greet(\\\"Zoë\\\"))\"}" } ], "exception": null }, "rand:5:8": { "content": "\n\n", "reasoning_content": "Write the script.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"greet.py\", \"content\": \"def greet(name: str) -> str:\\n \\\"\\\"\\\"Say hi.\\\"\\\"\\\"\\n return f\\\"Hello, {name}!\\\"\\n\\n\\nprint(greet(\\\"Zoë\\\"))\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/q38-whitespace-significant", "family": "qwen3-xml", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Replace the block exactly.', got 'Replace the block exactly.\\n'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Replace the block exactly.', got 'Replace the block exactly.\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Replace the block exactly.', got 'Replace the block exactly.\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Replace the block exactly.', got 'Replace the block exactly.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Replace the block exactly.', got 'Replace the block exactly.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Replace the block exactly.', got 'Replace the block exactly.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Replace the block exactly.', got 'Replace the block exactly.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Replace the block exactly.', got 'Replace the block exactly.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Replace the block exactly.', got 'Replace the block exactly.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.8-27B", "tokenizer": { "repo": "Qwen/Qwen3.8-27B", "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 248045, 74455, 198, 248068, 198 ] }, "stop": { "token": "<|im_end|>", "id": 248046, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "\n\n", "reasoning_content": "Replace the block exactly.\n", "tool_calls": [ { "name": "edit_file", "arguments_raw": "{\"path\": \"a.py\", \"old_string\": \" if x:\\n return 1\\n\", \"new_string\": \" two spaces \"}" } ], "exception": null }, "streams": { "one": { "content": "\n\n", "reasoning_content": "Replace the block exactly.\n", "tool_calls": [ { "name": "edit_file", "arguments_raw": "{\"path\": \"a.py\", \"old_string\": \" if x:\\n return 1\\n\", \"new_string\": \" two spaces \"}" } ], "exception": null }, "special": { "content": "\n\n", "reasoning_content": "Replace the block exactly.\n", "tool_calls": [ { "name": "edit_file", "arguments_raw": "{\"path\": \"a.py\", \"old_string\": \" if x:\\n return 1\\n\", \"new_string\": \" two spaces \"}" } ], "exception": null }, "token": { "content": "\n\n", "reasoning_content": "Replace the block exactly.\n", "tool_calls": [ { "name": "edit_file", "arguments_raw": "{\"path\": \"a.py\", \"old_string\": \" if x:\\n return 1\\n\", \"new_string\": \" two spaces \"}" } ], "exception": null }, "rand:1:8": { "content": "\n\n", "reasoning_content": "Replace the block exactly.\n", "tool_calls": [ { "name": "edit_file", "arguments_raw": "{\"path\": \"a.py\", \"old_string\": \" if x:\\n return 1\\n\", \"new_string\": \" two spaces \"}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": "Replace the block exactly.\n", "tool_calls": [ { "name": "edit_file", "arguments_raw": "{\"path\": \"a.py\", \"old_string\": \" if x:\\n return 1\\n\", \"new_string\": \" two spaces \"}" } ], "exception": null }, "rand:3:8": { "content": "\n\n", "reasoning_content": "Replace the block exactly.\n", "tool_calls": [ { "name": "edit_file", "arguments_raw": "{\"path\": \"a.py\", \"old_string\": \" if x:\\n return 1\\n\", \"new_string\": \" two spaces \"}" } ], "exception": null }, "rand:4:8": { "content": "\n\n", "reasoning_content": "Replace the block exactly.\n", "tool_calls": [ { "name": "edit_file", "arguments_raw": "{\"path\": \"a.py\", \"old_string\": \" if x:\\n return 1\\n\", \"new_string\": \" two spaces \"}" } ], "exception": null }, "rand:5:8": { "content": "\n\n", "reasoning_content": "Replace the block exactly.\n", "tool_calls": [ { "name": "edit_file", "arguments_raw": "{\"path\": \"a.py\", \"old_string\": \" if x:\\n return 1\\n\", \"new_string\": \" two spaces \"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/q38-numeric-and-literal-arguments", "family": "qwen3-xml", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Set the alarm.', got 'Set the alarm.\\n'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Set the alarm.', got 'Set the alarm.\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Set the alarm.', got 'Set the alarm.\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Set the alarm.', got 'Set the alarm.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Set the alarm.', got 'Set the alarm.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Set the alarm.', got 'Set the alarm.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Set the alarm.', got 'Set the alarm.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Set the alarm.', got 'Set the alarm.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Set the alarm.', got 'Set the alarm.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.8-27B", "tokenizer": { "repo": "Qwen/Qwen3.8-27B", "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 248045, 74455, 198, 248068, 198 ] }, "stop": { "token": "<|im_end|>", "id": 248046, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "\n\n", "reasoning_content": "Set the alarm.\n", "tool_calls": [ { "name": "set_alarm", "arguments_raw": "{\"hour\": 7, \"minute\": 30, \"volume\": 0.75, \"repeat\": true, \"label\": null}" } ], "exception": null }, "streams": { "one": { "content": "\n\n", "reasoning_content": "Set the alarm.\n", "tool_calls": [ { "name": "set_alarm", "arguments_raw": "{\"hour\": 7, \"minute\": 30, \"volume\": 0.75, \"repeat\": true, \"label\": null}" } ], "exception": null }, "special": { "content": "\n\n", "reasoning_content": "Set the alarm.\n", "tool_calls": [ { "name": "set_alarm", "arguments_raw": "{\"hour\": 7, \"minute\": 30, \"volume\": 0.75, \"repeat\": true, \"label\": null}" } ], "exception": null }, "token": { "content": "\n\n", "reasoning_content": "Set the alarm.\n", "tool_calls": [ { "name": "set_alarm", "arguments_raw": "{\"hour\": 7, \"minute\": 30, \"volume\": 0.75, \"repeat\": true, \"label\": null}" } ], "exception": null }, "rand:1:8": { "content": "\n\n", "reasoning_content": "Set the alarm.\n", "tool_calls": [ { "name": "set_alarm", "arguments_raw": "{\"hour\": 7, \"minute\": 30, \"volume\": 0.75, \"repeat\": true, \"label\": null}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": "Set the alarm.\n", "tool_calls": [ { "name": "set_alarm", "arguments_raw": "{\"hour\": 7, \"minute\": 30, \"volume\": 0.75, \"repeat\": true, \"label\": null}" } ], "exception": null }, "rand:3:8": { "content": "\n\n", "reasoning_content": "Set the alarm.\n", "tool_calls": [ { "name": "set_alarm", "arguments_raw": "{\"hour\": 7, \"minute\": 30, \"volume\": 0.75, \"repeat\": true, \"label\": null}" } ], "exception": null }, "rand:4:8": { "content": "\n\n", "reasoning_content": "Set the alarm.\n", "tool_calls": [ { "name": "set_alarm", "arguments_raw": "{\"hour\": 7, \"minute\": 30, \"volume\": 0.75, \"repeat\": true, \"label\": null}" } ], "exception": null }, "rand:5:8": { "content": "\n\n", "reasoning_content": "Set the alarm.\n", "tool_calls": [ { "name": "set_alarm", "arguments_raw": "{\"hour\": 7, \"minute\": 30, \"volume\": 0.75, \"repeat\": true, \"label\": null}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/q38-numeric-looking-strings", "family": "qwen3-xml", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Look it up.', got 'Look it up.\\n'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Look it up.', got 'Look it up.\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Look it up.', got 'Look it up.\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Look it up.', got 'Look it up.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Look it up.', got 'Look it up.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Look it up.', got 'Look it up.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Look it up.', got 'Look it up.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Look it up.', got 'Look it up.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Look it up.', got 'Look it up.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.8-27B", "tokenizer": { "repo": "Qwen/Qwen3.8-27B", "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 248045, 74455, 198, 248068, 198 ] }, "stop": { "token": "<|im_end|>", "id": 248046, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "\n\n", "reasoning_content": "Look it up.\n", "tool_calls": [ { "name": "lookup", "arguments_raw": "{\"zip\": \"02139\", \"account_id\": \"123456789012345678901\", \"flag\": \"true\", \"payload\": \"{\\\"a\\\": 1}\"}" } ], "exception": null }, "streams": { "one": { "content": "\n\n", "reasoning_content": "Look it up.\n", "tool_calls": [ { "name": "lookup", "arguments_raw": "{\"zip\": \"02139\", \"account_id\": \"123456789012345678901\", \"flag\": \"true\", \"payload\": \"{\\\"a\\\": 1}\"}" } ], "exception": null }, "special": { "content": "\n\n", "reasoning_content": "Look it up.\n", "tool_calls": [ { "name": "lookup", "arguments_raw": "{\"zip\": \"02139\", \"account_id\": \"123456789012345678901\", \"flag\": \"true\", \"payload\": \"{\\\"a\\\": 1}\"}" } ], "exception": null }, "token": { "content": "\n\n", "reasoning_content": "Look it up.\n", "tool_calls": [ { "name": "lookup", "arguments_raw": "{\"zip\": \"02139\", \"account_id\": \"123456789012345678901\", \"flag\": \"true\", \"payload\": \"{\\\"a\\\": 1}\"}" } ], "exception": null }, "rand:1:8": { "content": "\n\n", "reasoning_content": "Look it up.\n", "tool_calls": [ { "name": "lookup", "arguments_raw": "{\"zip\": \"02139\", \"account_id\": \"123456789012345678901\", \"flag\": \"true\", \"payload\": \"{\\\"a\\\": 1}\"}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": "Look it up.\n", "tool_calls": [ { "name": "lookup", "arguments_raw": "{\"zip\": \"02139\", \"account_id\": \"123456789012345678901\", \"flag\": \"true\", \"payload\": \"{\\\"a\\\": 1}\"}" } ], "exception": null }, "rand:3:8": { "content": "\n\n", "reasoning_content": "Look it up.\n", "tool_calls": [ { "name": "lookup", "arguments_raw": "{\"zip\": \"02139\", \"account_id\": \"123456789012345678901\", \"flag\": \"true\", \"payload\": \"{\\\"a\\\": 1}\"}" } ], "exception": null }, "rand:4:8": { "content": "\n\n", "reasoning_content": "Look it up.\n", "tool_calls": [ { "name": "lookup", "arguments_raw": "{\"zip\": \"02139\", \"account_id\": \"123456789012345678901\", \"flag\": \"true\", \"payload\": \"{\\\"a\\\": 1}\"}" } ], "exception": null }, "rand:5:8": { "content": "\n\n", "reasoning_content": "Look it up.\n", "tool_calls": [ { "name": "lookup", "arguments_raw": "{\"zip\": \"02139\", \"account_id\": \"123456789012345678901\", \"flag\": \"true\", \"payload\": \"{\\\"a\\\": 1}\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/q38-string-escapes", "family": "qwen3-xml", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write it.', got 'Write it.\\n'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write it.', got 'Write it.\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write it.', got 'Write it.\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write it.', got 'Write it.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write it.', got 'Write it.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write it.', got 'Write it.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write it.', got 'Write it.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write it.', got 'Write it.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write it.', got 'Write it.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.8-27B", "tokenizer": { "repo": "Qwen/Qwen3.8-27B", "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 248045, 74455, 198, 248068, 198 ] }, "stop": { "token": "<|im_end|>", "id": 248046, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "\n\n", "reasoning_content": "Write it.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"C:\\\\tmp\\\\x.txt\", \"content\": \"a \\\"quote\\\", a \\\\backslash, & ampersand\\ttab\"}" } ], "exception": null }, "streams": { "one": { "content": "\n\n", "reasoning_content": "Write it.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"C:\\\\tmp\\\\x.txt\", \"content\": \"a \\\"quote\\\", a \\\\backslash, & ampersand\\ttab\"}" } ], "exception": null }, "special": { "content": "\n\n", "reasoning_content": "Write it.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"C:\\\\tmp\\\\x.txt\", \"content\": \"a \\\"quote\\\", a \\\\backslash, & ampersand\\ttab\"}" } ], "exception": null }, "token": { "content": "\n\n", "reasoning_content": "Write it.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"C:\\\\tmp\\\\x.txt\", \"content\": \"a \\\"quote\\\", a \\\\backslash, & ampersand\\ttab\"}" } ], "exception": null }, "rand:1:8": { "content": "\n\n", "reasoning_content": "Write it.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"C:\\\\tmp\\\\x.txt\", \"content\": \"a \\\"quote\\\", a \\\\backslash, & ampersand\\ttab\"}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": "Write it.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"C:\\\\tmp\\\\x.txt\", \"content\": \"a \\\"quote\\\", a \\\\backslash, & ampersand\\ttab\"}" } ], "exception": null }, "rand:3:8": { "content": "\n\n", "reasoning_content": "Write it.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"C:\\\\tmp\\\\x.txt\", \"content\": \"a \\\"quote\\\", a \\\\backslash, & ampersand\\ttab\"}" } ], "exception": null }, "rand:4:8": { "content": "\n\n", "reasoning_content": "Write it.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"C:\\\\tmp\\\\x.txt\", \"content\": \"a \\\"quote\\\", a \\\\backslash, & ampersand\\ttab\"}" } ], "exception": null }, "rand:5:8": { "content": "\n\n", "reasoning_content": "Write it.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"C:\\\\tmp\\\\x.txt\", \"content\": \"a \\\"quote\\\", a \\\\backslash, & ampersand\\ttab\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/q38-long-arguments", "family": "qwen3-xml", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write the module.', got 'Write the module.\\n'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write the module.', got 'Write the module.\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write the module.', got 'Write the module.\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write the module.', got 'Write the module.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write the module.', got 'Write the module.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write the module.', got 'Write the module.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write the module.', got 'Write the module.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write the module.', got 'Write the module.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Write the module.', got 'Write the module.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.8-27B", "tokenizer": { "repo": "Qwen/Qwen3.8-27B", "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 248045, 74455, 198, 248068, 198 ] }, "stop": { "token": "<|im_end|>", "id": 248046, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "\n\n", "reasoning_content": "Write the module.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"steps.py\", \"content\": \"\\\"\\\"\\\"Generated module used as a long tool argument.\\\"\\\"\\\"\\n\\nfrom __future__ import annotations\\n\\ndef step_00(x: int) -> int:\\n return x + 0\\n\\ndef step_01(x: int) -> int:\\n return x + 1\\n\\ndef step_02(x: int) -> int:\\n return x + 2\\n\\ndef step_03(x: int) -> int:\\n return x + 3\\n\\ndef step_04(x: int) -> int:\\n return x + 4\\n\\ndef step_05(x: int) -> int:\\n return x + 5\\n\\ndef step_06(x: int) -> int:\\n return x + 6\\n\\ndef step_07(x: int) -> int:\\n return x + 7\\n\\ndef step_08(x: int) -> int:\\n return x + 8\\n\\ndef step_09(x: int) -> int:\\n return x + 9\\n\\ndef step_10(x: int) -> int:\\n return x + 10\\n\\ndef step_11(x: int) -> int:\\n return x + 11\\n\\ndef step_12(x: int) -> int:\\n return x + 12\\n\\ndef step_13(x: int) -> int:\\n return x + 13\\n\\ndef step_14(x: int) -> int:\\n return x + 14\\n\\ndef step_15(x: int) -> int:\\n return x + 15\\n\\ndef step_16(x: int) -> int:\\n return x + 16\\n\\ndef step_17(x: int) -> int:\\n return x + 17\\n\\ndef step_18(x: int) -> int:\\n return x + 18\\n\\ndef step_19(x: int) -> int:\\n return x + 19\\n\\ndef step_20(x: int) -> int:\\n return x + 20\\n\\ndef step_21(x: int) -> int:\\n return x + 21\\n\\ndef step_22(x: int) -> int:\\n return x + 22\\n\\ndef step_23(x: int) -> int:\\n return x + 23\\n\\ndef step_24(x: int) -> int:\\n return x + 24\\n\\ndef step_25(x: int) -> int:\\n return x + 25\\n\\ndef step_26(x: int) -> int:\\n return x + 26\\n\\ndef step_27(x: int) -> int:\\n return x + 27\\n\\ndef step_28(x: int) -> int:\\n return x + 28\\n\\ndef step_29(x: int) -> int:\\n return x + 29\\n\\ndef step_30(x: int) -> int:\\n return x + 30\\n\\ndef step_31(x: int) -> int:\\n return x + 31\\n\\ndef step_32(x: int) -> int:\\n return x + 32\\n\\ndef step_33(x: int) -> int:\\n return x + 33\\n\\ndef step_34(x: int) -> int:\\n return x + 34\\n\\ndef step_35(x: int) -> int:\\n return x + 35\\n\\ndef step_36(x: int) -> int:\\n return x + 36\\n\\ndef step_37(x: int) -> int:\\n return x + 37\\n\\ndef step_38(x: int) -> int:\\n return x + 38\\n\\ndef step_39(x: int) -> int:\\n return x + 39\\n\\ndef step_40(x: int) -> int:\\n return x + 40\\n\\ndef step_41(x: int) -> int:\\n return x + 41\\n\\ndef step_42(x: int) -> int:\\n return x + 42\\n\\ndef step_43(x: int) -> int:\\n return x + 43\\n\\ndef step_44(x: int) -> int:\\n return x + 44\\n\\ndef step_45(x: int) -> int:\\n return x + 45\\n\\ndef step_46(x: int) -> int:\\n return x + 46\\n\\ndef step_47(x: int) -> int:\\n return x + 47\\n\\ndef step_48(x: int) -> int:\\n return x + 48\\n\\ndef step_49(x: int) -> int:\\n return x + 49\\n\\ndef step_50(x: int) -> int:\\n return x + 50\\n\\ndef step_51(x: int) -> int:\\n return x + 51\\n\\ndef step_52(x: int) -> int:\\n return x + 52\\n\\ndef step_53(x: int) -> int:\\n return x + 53\\n\\ndef step_54(x: int) -> int:\\n return x + 54\\n\\ndef step_55(x: int) -> int:\\n return x + 55\\n\\ndef step_56(x: int) -> int:\\n return x + 56\\n\\ndef step_57(x: int) -> int:\\n return x + 57\\n\\ndef step_58(x: int) -> int:\\n return x + 58\\n\\ndef step_59(x: int) -> int:\\n return x + 59\\n\\ndef step_60(x: int) -> int:\\n return x + 60\\n\\ndef step_61(x: int) -> int:\\n return x + 61\\n\\ndef step_62(x: int) -> int:\\n return x + 62\\n\\ndef step_63(x: int) -> int:\\n return x + 63\\n\\ndef step_64(x: int) -> int:\\n return x + 64\\n\\ndef step_65(x: int) -> int:\\n return x + 65\\n\\ndef step_66(x: int) -> int:\\n return x + 66\\n\\ndef step_67(x: int) -> int:\\n return x + 67\\n\\ndef step_68(x: int) -> int:\\n return x + 68\\n\\ndef step_69(x: int) -> int:\\n return x + 69\\n\\ndef step_70(x: int) -> int:\\n return x + 70\\n\\ndef step_71(x: int) -> int:\\n return x + 71\\n\\ndef step_72(x: int) -> int:\\n return x + 72\\n\\ndef step_73(x: int) -> int:\\n return x + 73\\n\\ndef step_74(x: int) -> int:\\n return x + 74\\n\\ndef step_75(x: int) -> int:\\n return x + 75\\n\\ndef step_76(x: int) -> int:\\n return x + 76\\n\\ndef step_77(x: int) -> int:\\n return x + 77\\n\\ndef step_78(x: int) -> int:\\n return x + 78\\n\\ndef step_79(x: int) -> int:\\n return x + 79\\n\"}" } ], "exception": null }, "streams": { "one": { "content": "\n\n", "reasoning_content": "Write the module.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"steps.py\", \"content\": \"\\\"\\\"\\\"Generated module used as a long tool argument.\\\"\\\"\\\"\\n\\nfrom __future__ import annotations\\n\\ndef step_00(x: int) -> int:\\n return x + 0\\n\\ndef step_01(x: int) -> int:\\n return x + 1\\n\\ndef step_02(x: int) -> int:\\n return x + 2\\n\\ndef step_03(x: int) -> int:\\n return x + 3\\n\\ndef step_04(x: int) -> int:\\n return x + 4\\n\\ndef step_05(x: int) -> int:\\n return x + 5\\n\\ndef step_06(x: int) -> int:\\n return x + 6\\n\\ndef step_07(x: int) -> int:\\n return x + 7\\n\\ndef step_08(x: int) -> int:\\n return x + 8\\n\\ndef step_09(x: int) -> int:\\n return x + 9\\n\\ndef step_10(x: int) -> int:\\n return x + 10\\n\\ndef step_11(x: int) -> int:\\n return x + 11\\n\\ndef step_12(x: int) -> int:\\n return x + 12\\n\\ndef step_13(x: int) -> int:\\n return x + 13\\n\\ndef step_14(x: int) -> int:\\n return x + 14\\n\\ndef step_15(x: int) -> int:\\n return x + 15\\n\\ndef step_16(x: int) -> int:\\n return x + 16\\n\\ndef step_17(x: int) -> int:\\n return x + 17\\n\\ndef step_18(x: int) -> int:\\n return x + 18\\n\\ndef step_19(x: int) -> int:\\n return x + 19\\n\\ndef step_20(x: int) -> int:\\n return x + 20\\n\\ndef step_21(x: int) -> int:\\n return x + 21\\n\\ndef step_22(x: int) -> int:\\n return x + 22\\n\\ndef step_23(x: int) -> int:\\n return x + 23\\n\\ndef step_24(x: int) -> int:\\n return x + 24\\n\\ndef step_25(x: int) -> int:\\n return x + 25\\n\\ndef step_26(x: int) -> int:\\n return x + 26\\n\\ndef step_27(x: int) -> int:\\n return x + 27\\n\\ndef step_28(x: int) -> int:\\n return x + 28\\n\\ndef step_29(x: int) -> int:\\n return x + 29\\n\\ndef step_30(x: int) -> int:\\n return x + 30\\n\\ndef step_31(x: int) -> int:\\n return x + 31\\n\\ndef step_32(x: int) -> int:\\n return x + 32\\n\\ndef step_33(x: int) -> int:\\n return x + 33\\n\\ndef step_34(x: int) -> int:\\n return x + 34\\n\\ndef step_35(x: int) -> int:\\n return x + 35\\n\\ndef step_36(x: int) -> int:\\n return x + 36\\n\\ndef step_37(x: int) -> int:\\n return x + 37\\n\\ndef step_38(x: int) -> int:\\n return x + 38\\n\\ndef step_39(x: int) -> int:\\n return x + 39\\n\\ndef step_40(x: int) -> int:\\n return x + 40\\n\\ndef step_41(x: int) -> int:\\n return x + 41\\n\\ndef step_42(x: int) -> int:\\n return x + 42\\n\\ndef step_43(x: int) -> int:\\n return x + 43\\n\\ndef step_44(x: int) -> int:\\n return x + 44\\n\\ndef step_45(x: int) -> int:\\n return x + 45\\n\\ndef step_46(x: int) -> int:\\n return x + 46\\n\\ndef step_47(x: int) -> int:\\n return x + 47\\n\\ndef step_48(x: int) -> int:\\n return x + 48\\n\\ndef step_49(x: int) -> int:\\n return x + 49\\n\\ndef step_50(x: int) -> int:\\n return x + 50\\n\\ndef step_51(x: int) -> int:\\n return x + 51\\n\\ndef step_52(x: int) -> int:\\n return x + 52\\n\\ndef step_53(x: int) -> int:\\n return x + 53\\n\\ndef step_54(x: int) -> int:\\n return x + 54\\n\\ndef step_55(x: int) -> int:\\n return x + 55\\n\\ndef step_56(x: int) -> int:\\n return x + 56\\n\\ndef step_57(x: int) -> int:\\n return x + 57\\n\\ndef step_58(x: int) -> int:\\n return x + 58\\n\\ndef step_59(x: int) -> int:\\n return x + 59\\n\\ndef step_60(x: int) -> int:\\n return x + 60\\n\\ndef step_61(x: int) -> int:\\n return x + 61\\n\\ndef step_62(x: int) -> int:\\n return x + 62\\n\\ndef step_63(x: int) -> int:\\n return x + 63\\n\\ndef step_64(x: int) -> int:\\n return x + 64\\n\\ndef step_65(x: int) -> int:\\n return x + 65\\n\\ndef step_66(x: int) -> int:\\n return x + 66\\n\\ndef step_67(x: int) -> int:\\n return x + 67\\n\\ndef step_68(x: int) -> int:\\n return x + 68\\n\\ndef step_69(x: int) -> int:\\n return x + 69\\n\\ndef step_70(x: int) -> int:\\n return x + 70\\n\\ndef step_71(x: int) -> int:\\n return x + 71\\n\\ndef step_72(x: int) -> int:\\n return x + 72\\n\\ndef step_73(x: int) -> int:\\n return x + 73\\n\\ndef step_74(x: int) -> int:\\n return x + 74\\n\\ndef step_75(x: int) -> int:\\n return x + 75\\n\\ndef step_76(x: int) -> int:\\n return x + 76\\n\\ndef step_77(x: int) -> int:\\n return x + 77\\n\\ndef step_78(x: int) -> int:\\n return x + 78\\n\\ndef step_79(x: int) -> int:\\n return x + 79\\n\"}" } ], "exception": null }, "special": { "content": "\n\n", "reasoning_content": "Write the module.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"steps.py\", \"content\": \"\\\"\\\"\\\"Generated module used as a long tool argument.\\\"\\\"\\\"\\n\\nfrom __future__ import annotations\\n\\ndef step_00(x: int) -> int:\\n return x + 0\\n\\ndef step_01(x: int) -> int:\\n return x + 1\\n\\ndef step_02(x: int) -> int:\\n return x + 2\\n\\ndef step_03(x: int) -> int:\\n return x + 3\\n\\ndef step_04(x: int) -> int:\\n return x + 4\\n\\ndef step_05(x: int) -> int:\\n return x + 5\\n\\ndef step_06(x: int) -> int:\\n return x + 6\\n\\ndef step_07(x: int) -> int:\\n return x + 7\\n\\ndef step_08(x: int) -> int:\\n return x + 8\\n\\ndef step_09(x: int) -> int:\\n return x + 9\\n\\ndef step_10(x: int) -> int:\\n return x + 10\\n\\ndef step_11(x: int) -> int:\\n return x + 11\\n\\ndef step_12(x: int) -> int:\\n return x + 12\\n\\ndef step_13(x: int) -> int:\\n return x + 13\\n\\ndef step_14(x: int) -> int:\\n return x + 14\\n\\ndef step_15(x: int) -> int:\\n return x + 15\\n\\ndef step_16(x: int) -> int:\\n return x + 16\\n\\ndef step_17(x: int) -> int:\\n return x + 17\\n\\ndef step_18(x: int) -> int:\\n return x + 18\\n\\ndef step_19(x: int) -> int:\\n return x + 19\\n\\ndef step_20(x: int) -> int:\\n return x + 20\\n\\ndef step_21(x: int) -> int:\\n return x + 21\\n\\ndef step_22(x: int) -> int:\\n return x + 22\\n\\ndef step_23(x: int) -> int:\\n return x + 23\\n\\ndef step_24(x: int) -> int:\\n return x + 24\\n\\ndef step_25(x: int) -> int:\\n return x + 25\\n\\ndef step_26(x: int) -> int:\\n return x + 26\\n\\ndef step_27(x: int) -> int:\\n return x + 27\\n\\ndef step_28(x: int) -> int:\\n return x + 28\\n\\ndef step_29(x: int) -> int:\\n return x + 29\\n\\ndef step_30(x: int) -> int:\\n return x + 30\\n\\ndef step_31(x: int) -> int:\\n return x + 31\\n\\ndef step_32(x: int) -> int:\\n return x + 32\\n\\ndef step_33(x: int) -> int:\\n return x + 33\\n\\ndef step_34(x: int) -> int:\\n return x + 34\\n\\ndef step_35(x: int) -> int:\\n return x + 35\\n\\ndef step_36(x: int) -> int:\\n return x + 36\\n\\ndef step_37(x: int) -> int:\\n return x + 37\\n\\ndef step_38(x: int) -> int:\\n return x + 38\\n\\ndef step_39(x: int) -> int:\\n return x + 39\\n\\ndef step_40(x: int) -> int:\\n return x + 40\\n\\ndef step_41(x: int) -> int:\\n return x + 41\\n\\ndef step_42(x: int) -> int:\\n return x + 42\\n\\ndef step_43(x: int) -> int:\\n return x + 43\\n\\ndef step_44(x: int) -> int:\\n return x + 44\\n\\ndef step_45(x: int) -> int:\\n return x + 45\\n\\ndef step_46(x: int) -> int:\\n return x + 46\\n\\ndef step_47(x: int) -> int:\\n return x + 47\\n\\ndef step_48(x: int) -> int:\\n return x + 48\\n\\ndef step_49(x: int) -> int:\\n return x + 49\\n\\ndef step_50(x: int) -> int:\\n return x + 50\\n\\ndef step_51(x: int) -> int:\\n return x + 51\\n\\ndef step_52(x: int) -> int:\\n return x + 52\\n\\ndef step_53(x: int) -> int:\\n return x + 53\\n\\ndef step_54(x: int) -> int:\\n return x + 54\\n\\ndef step_55(x: int) -> int:\\n return x + 55\\n\\ndef step_56(x: int) -> int:\\n return x + 56\\n\\ndef step_57(x: int) -> int:\\n return x + 57\\n\\ndef step_58(x: int) -> int:\\n return x + 58\\n\\ndef step_59(x: int) -> int:\\n return x + 59\\n\\ndef step_60(x: int) -> int:\\n return x + 60\\n\\ndef step_61(x: int) -> int:\\n return x + 61\\n\\ndef step_62(x: int) -> int:\\n return x + 62\\n\\ndef step_63(x: int) -> int:\\n return x + 63\\n\\ndef step_64(x: int) -> int:\\n return x + 64\\n\\ndef step_65(x: int) -> int:\\n return x + 65\\n\\ndef step_66(x: int) -> int:\\n return x + 66\\n\\ndef step_67(x: int) -> int:\\n return x + 67\\n\\ndef step_68(x: int) -> int:\\n return x + 68\\n\\ndef step_69(x: int) -> int:\\n return x + 69\\n\\ndef step_70(x: int) -> int:\\n return x + 70\\n\\ndef step_71(x: int) -> int:\\n return x + 71\\n\\ndef step_72(x: int) -> int:\\n return x + 72\\n\\ndef step_73(x: int) -> int:\\n return x + 73\\n\\ndef step_74(x: int) -> int:\\n return x + 74\\n\\ndef step_75(x: int) -> int:\\n return x + 75\\n\\ndef step_76(x: int) -> int:\\n return x + 76\\n\\ndef step_77(x: int) -> int:\\n return x + 77\\n\\ndef step_78(x: int) -> int:\\n return x + 78\\n\\ndef step_79(x: int) -> int:\\n return x + 79\\n\"}" } ], "exception": null }, "token": { "content": "\n\n", "reasoning_content": "Write the module.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"steps.py\", \"content\": \"\\\"\\\"\\\"Generated module used as a long tool argument.\\\"\\\"\\\"\\n\\nfrom __future__ import annotations\\n\\ndef step_00(x: int) -> int:\\n return x + 0\\n\\ndef step_01(x: int) -> int:\\n return x + 1\\n\\ndef step_02(x: int) -> int:\\n return x + 2\\n\\ndef step_03(x: int) -> int:\\n return x + 3\\n\\ndef step_04(x: int) -> int:\\n return x + 4\\n\\ndef step_05(x: int) -> int:\\n return x + 5\\n\\ndef step_06(x: int) -> int:\\n return x + 6\\n\\ndef step_07(x: int) -> int:\\n return x + 7\\n\\ndef step_08(x: int) -> int:\\n return x + 8\\n\\ndef step_09(x: int) -> int:\\n return x + 9\\n\\ndef step_10(x: int) -> int:\\n return x + 10\\n\\ndef step_11(x: int) -> int:\\n return x + 11\\n\\ndef step_12(x: int) -> int:\\n return x + 12\\n\\ndef step_13(x: int) -> int:\\n return x + 13\\n\\ndef step_14(x: int) -> int:\\n return x + 14\\n\\ndef step_15(x: int) -> int:\\n return x + 15\\n\\ndef step_16(x: int) -> int:\\n return x + 16\\n\\ndef step_17(x: int) -> int:\\n return x + 17\\n\\ndef step_18(x: int) -> int:\\n return x + 18\\n\\ndef step_19(x: int) -> int:\\n return x + 19\\n\\ndef step_20(x: int) -> int:\\n return x + 20\\n\\ndef step_21(x: int) -> int:\\n return x + 21\\n\\ndef step_22(x: int) -> int:\\n return x + 22\\n\\ndef step_23(x: int) -> int:\\n return x + 23\\n\\ndef step_24(x: int) -> int:\\n return x + 24\\n\\ndef step_25(x: int) -> int:\\n return x + 25\\n\\ndef step_26(x: int) -> int:\\n return x + 26\\n\\ndef step_27(x: int) -> int:\\n return x + 27\\n\\ndef step_28(x: int) -> int:\\n return x + 28\\n\\ndef step_29(x: int) -> int:\\n return x + 29\\n\\ndef step_30(x: int) -> int:\\n return x + 30\\n\\ndef step_31(x: int) -> int:\\n return x + 31\\n\\ndef step_32(x: int) -> int:\\n return x + 32\\n\\ndef step_33(x: int) -> int:\\n return x + 33\\n\\ndef step_34(x: int) -> int:\\n return x + 34\\n\\ndef step_35(x: int) -> int:\\n return x + 35\\n\\ndef step_36(x: int) -> int:\\n return x + 36\\n\\ndef step_37(x: int) -> int:\\n return x + 37\\n\\ndef step_38(x: int) -> int:\\n return x + 38\\n\\ndef step_39(x: int) -> int:\\n return x + 39\\n\\ndef step_40(x: int) -> int:\\n return x + 40\\n\\ndef step_41(x: int) -> int:\\n return x + 41\\n\\ndef step_42(x: int) -> int:\\n return x + 42\\n\\ndef step_43(x: int) -> int:\\n return x + 43\\n\\ndef step_44(x: int) -> int:\\n return x + 44\\n\\ndef step_45(x: int) -> int:\\n return x + 45\\n\\ndef step_46(x: int) -> int:\\n return x + 46\\n\\ndef step_47(x: int) -> int:\\n return x + 47\\n\\ndef step_48(x: int) -> int:\\n return x + 48\\n\\ndef step_49(x: int) -> int:\\n return x + 49\\n\\ndef step_50(x: int) -> int:\\n return x + 50\\n\\ndef step_51(x: int) -> int:\\n return x + 51\\n\\ndef step_52(x: int) -> int:\\n return x + 52\\n\\ndef step_53(x: int) -> int:\\n return x + 53\\n\\ndef step_54(x: int) -> int:\\n return x + 54\\n\\ndef step_55(x: int) -> int:\\n return x + 55\\n\\ndef step_56(x: int) -> int:\\n return x + 56\\n\\ndef step_57(x: int) -> int:\\n return x + 57\\n\\ndef step_58(x: int) -> int:\\n return x + 58\\n\\ndef step_59(x: int) -> int:\\n return x + 59\\n\\ndef step_60(x: int) -> int:\\n return x + 60\\n\\ndef step_61(x: int) -> int:\\n return x + 61\\n\\ndef step_62(x: int) -> int:\\n return x + 62\\n\\ndef step_63(x: int) -> int:\\n return x + 63\\n\\ndef step_64(x: int) -> int:\\n return x + 64\\n\\ndef step_65(x: int) -> int:\\n return x + 65\\n\\ndef step_66(x: int) -> int:\\n return x + 66\\n\\ndef step_67(x: int) -> int:\\n return x + 67\\n\\ndef step_68(x: int) -> int:\\n return x + 68\\n\\ndef step_69(x: int) -> int:\\n return x + 69\\n\\ndef step_70(x: int) -> int:\\n return x + 70\\n\\ndef step_71(x: int) -> int:\\n return x + 71\\n\\ndef step_72(x: int) -> int:\\n return x + 72\\n\\ndef step_73(x: int) -> int:\\n return x + 73\\n\\ndef step_74(x: int) -> int:\\n return x + 74\\n\\ndef step_75(x: int) -> int:\\n return x + 75\\n\\ndef step_76(x: int) -> int:\\n return x + 76\\n\\ndef step_77(x: int) -> int:\\n return x + 77\\n\\ndef step_78(x: int) -> int:\\n return x + 78\\n\\ndef step_79(x: int) -> int:\\n return x + 79\\n\"}" } ], "exception": null }, "rand:1:8": { "content": "\n\n", "reasoning_content": "Write the module.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"steps.py\", \"content\": \"\\\"\\\"\\\"Generated module used as a long tool argument.\\\"\\\"\\\"\\n\\nfrom __future__ import annotations\\n\\ndef step_00(x: int) -> int:\\n return x + 0\\n\\ndef step_01(x: int) -> int:\\n return x + 1\\n\\ndef step_02(x: int) -> int:\\n return x + 2\\n\\ndef step_03(x: int) -> int:\\n return x + 3\\n\\ndef step_04(x: int) -> int:\\n return x + 4\\n\\ndef step_05(x: int) -> int:\\n return x + 5\\n\\ndef step_06(x: int) -> int:\\n return x + 6\\n\\ndef step_07(x: int) -> int:\\n return x + 7\\n\\ndef step_08(x: int) -> int:\\n return x + 8\\n\\ndef step_09(x: int) -> int:\\n return x + 9\\n\\ndef step_10(x: int) -> int:\\n return x + 10\\n\\ndef step_11(x: int) -> int:\\n return x + 11\\n\\ndef step_12(x: int) -> int:\\n return x + 12\\n\\ndef step_13(x: int) -> int:\\n return x + 13\\n\\ndef step_14(x: int) -> int:\\n return x + 14\\n\\ndef step_15(x: int) -> int:\\n return x + 15\\n\\ndef step_16(x: int) -> int:\\n return x + 16\\n\\ndef step_17(x: int) -> int:\\n return x + 17\\n\\ndef step_18(x: int) -> int:\\n return x + 18\\n\\ndef step_19(x: int) -> int:\\n return x + 19\\n\\ndef step_20(x: int) -> int:\\n return x + 20\\n\\ndef step_21(x: int) -> int:\\n return x + 21\\n\\ndef step_22(x: int) -> int:\\n return x + 22\\n\\ndef step_23(x: int) -> int:\\n return x + 23\\n\\ndef step_24(x: int) -> int:\\n return x + 24\\n\\ndef step_25(x: int) -> int:\\n return x + 25\\n\\ndef step_26(x: int) -> int:\\n return x + 26\\n\\ndef step_27(x: int) -> int:\\n return x + 27\\n\\ndef step_28(x: int) -> int:\\n return x + 28\\n\\ndef step_29(x: int) -> int:\\n return x + 29\\n\\ndef step_30(x: int) -> int:\\n return x + 30\\n\\ndef step_31(x: int) -> int:\\n return x + 31\\n\\ndef step_32(x: int) -> int:\\n return x + 32\\n\\ndef step_33(x: int) -> int:\\n return x + 33\\n\\ndef step_34(x: int) -> int:\\n return x + 34\\n\\ndef step_35(x: int) -> int:\\n return x + 35\\n\\ndef step_36(x: int) -> int:\\n return x + 36\\n\\ndef step_37(x: int) -> int:\\n return x + 37\\n\\ndef step_38(x: int) -> int:\\n return x + 38\\n\\ndef step_39(x: int) -> int:\\n return x + 39\\n\\ndef step_40(x: int) -> int:\\n return x + 40\\n\\ndef step_41(x: int) -> int:\\n return x + 41\\n\\ndef step_42(x: int) -> int:\\n return x + 42\\n\\ndef step_43(x: int) -> int:\\n return x + 43\\n\\ndef step_44(x: int) -> int:\\n return x + 44\\n\\ndef step_45(x: int) -> int:\\n return x + 45\\n\\ndef step_46(x: int) -> int:\\n return x + 46\\n\\ndef step_47(x: int) -> int:\\n return x + 47\\n\\ndef step_48(x: int) -> int:\\n return x + 48\\n\\ndef step_49(x: int) -> int:\\n return x + 49\\n\\ndef step_50(x: int) -> int:\\n return x + 50\\n\\ndef step_51(x: int) -> int:\\n return x + 51\\n\\ndef step_52(x: int) -> int:\\n return x + 52\\n\\ndef step_53(x: int) -> int:\\n return x + 53\\n\\ndef step_54(x: int) -> int:\\n return x + 54\\n\\ndef step_55(x: int) -> int:\\n return x + 55\\n\\ndef step_56(x: int) -> int:\\n return x + 56\\n\\ndef step_57(x: int) -> int:\\n return x + 57\\n\\ndef step_58(x: int) -> int:\\n return x + 58\\n\\ndef step_59(x: int) -> int:\\n return x + 59\\n\\ndef step_60(x: int) -> int:\\n return x + 60\\n\\ndef step_61(x: int) -> int:\\n return x + 61\\n\\ndef step_62(x: int) -> int:\\n return x + 62\\n\\ndef step_63(x: int) -> int:\\n return x + 63\\n\\ndef step_64(x: int) -> int:\\n return x + 64\\n\\ndef step_65(x: int) -> int:\\n return x + 65\\n\\ndef step_66(x: int) -> int:\\n return x + 66\\n\\ndef step_67(x: int) -> int:\\n return x + 67\\n\\ndef step_68(x: int) -> int:\\n return x + 68\\n\\ndef step_69(x: int) -> int:\\n return x + 69\\n\\ndef step_70(x: int) -> int:\\n return x + 70\\n\\ndef step_71(x: int) -> int:\\n return x + 71\\n\\ndef step_72(x: int) -> int:\\n return x + 72\\n\\ndef step_73(x: int) -> int:\\n return x + 73\\n\\ndef step_74(x: int) -> int:\\n return x + 74\\n\\ndef step_75(x: int) -> int:\\n return x + 75\\n\\ndef step_76(x: int) -> int:\\n return x + 76\\n\\ndef step_77(x: int) -> int:\\n return x + 77\\n\\ndef step_78(x: int) -> int:\\n return x + 78\\n\\ndef step_79(x: int) -> int:\\n return x + 79\\n\"}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": "Write the module.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"steps.py\", \"content\": \"\\\"\\\"\\\"Generated module used as a long tool argument.\\\"\\\"\\\"\\n\\nfrom __future__ import annotations\\n\\ndef step_00(x: int) -> int:\\n return x + 0\\n\\ndef step_01(x: int) -> int:\\n return x + 1\\n\\ndef step_02(x: int) -> int:\\n return x + 2\\n\\ndef step_03(x: int) -> int:\\n return x + 3\\n\\ndef step_04(x: int) -> int:\\n return x + 4\\n\\ndef step_05(x: int) -> int:\\n return x + 5\\n\\ndef step_06(x: int) -> int:\\n return x + 6\\n\\ndef step_07(x: int) -> int:\\n return x + 7\\n\\ndef step_08(x: int) -> int:\\n return x + 8\\n\\ndef step_09(x: int) -> int:\\n return x + 9\\n\\ndef step_10(x: int) -> int:\\n return x + 10\\n\\ndef step_11(x: int) -> int:\\n return x + 11\\n\\ndef step_12(x: int) -> int:\\n return x + 12\\n\\ndef step_13(x: int) -> int:\\n return x + 13\\n\\ndef step_14(x: int) -> int:\\n return x + 14\\n\\ndef step_15(x: int) -> int:\\n return x + 15\\n\\ndef step_16(x: int) -> int:\\n return x + 16\\n\\ndef step_17(x: int) -> int:\\n return x + 17\\n\\ndef step_18(x: int) -> int:\\n return x + 18\\n\\ndef step_19(x: int) -> int:\\n return x + 19\\n\\ndef step_20(x: int) -> int:\\n return x + 20\\n\\ndef step_21(x: int) -> int:\\n return x + 21\\n\\ndef step_22(x: int) -> int:\\n return x + 22\\n\\ndef step_23(x: int) -> int:\\n return x + 23\\n\\ndef step_24(x: int) -> int:\\n return x + 24\\n\\ndef step_25(x: int) -> int:\\n return x + 25\\n\\ndef step_26(x: int) -> int:\\n return x + 26\\n\\ndef step_27(x: int) -> int:\\n return x + 27\\n\\ndef step_28(x: int) -> int:\\n return x + 28\\n\\ndef step_29(x: int) -> int:\\n return x + 29\\n\\ndef step_30(x: int) -> int:\\n return x + 30\\n\\ndef step_31(x: int) -> int:\\n return x + 31\\n\\ndef step_32(x: int) -> int:\\n return x + 32\\n\\ndef step_33(x: int) -> int:\\n return x + 33\\n\\ndef step_34(x: int) -> int:\\n return x + 34\\n\\ndef step_35(x: int) -> int:\\n return x + 35\\n\\ndef step_36(x: int) -> int:\\n return x + 36\\n\\ndef step_37(x: int) -> int:\\n return x + 37\\n\\ndef step_38(x: int) -> int:\\n return x + 38\\n\\ndef step_39(x: int) -> int:\\n return x + 39\\n\\ndef step_40(x: int) -> int:\\n return x + 40\\n\\ndef step_41(x: int) -> int:\\n return x + 41\\n\\ndef step_42(x: int) -> int:\\n return x + 42\\n\\ndef step_43(x: int) -> int:\\n return x + 43\\n\\ndef step_44(x: int) -> int:\\n return x + 44\\n\\ndef step_45(x: int) -> int:\\n return x + 45\\n\\ndef step_46(x: int) -> int:\\n return x + 46\\n\\ndef step_47(x: int) -> int:\\n return x + 47\\n\\ndef step_48(x: int) -> int:\\n return x + 48\\n\\ndef step_49(x: int) -> int:\\n return x + 49\\n\\ndef step_50(x: int) -> int:\\n return x + 50\\n\\ndef step_51(x: int) -> int:\\n return x + 51\\n\\ndef step_52(x: int) -> int:\\n return x + 52\\n\\ndef step_53(x: int) -> int:\\n return x + 53\\n\\ndef step_54(x: int) -> int:\\n return x + 54\\n\\ndef step_55(x: int) -> int:\\n return x + 55\\n\\ndef step_56(x: int) -> int:\\n return x + 56\\n\\ndef step_57(x: int) -> int:\\n return x + 57\\n\\ndef step_58(x: int) -> int:\\n return x + 58\\n\\ndef step_59(x: int) -> int:\\n return x + 59\\n\\ndef step_60(x: int) -> int:\\n return x + 60\\n\\ndef step_61(x: int) -> int:\\n return x + 61\\n\\ndef step_62(x: int) -> int:\\n return x + 62\\n\\ndef step_63(x: int) -> int:\\n return x + 63\\n\\ndef step_64(x: int) -> int:\\n return x + 64\\n\\ndef step_65(x: int) -> int:\\n return x + 65\\n\\ndef step_66(x: int) -> int:\\n return x + 66\\n\\ndef step_67(x: int) -> int:\\n return x + 67\\n\\ndef step_68(x: int) -> int:\\n return x + 68\\n\\ndef step_69(x: int) -> int:\\n return x + 69\\n\\ndef step_70(x: int) -> int:\\n return x + 70\\n\\ndef step_71(x: int) -> int:\\n return x + 71\\n\\ndef step_72(x: int) -> int:\\n return x + 72\\n\\ndef step_73(x: int) -> int:\\n return x + 73\\n\\ndef step_74(x: int) -> int:\\n return x + 74\\n\\ndef step_75(x: int) -> int:\\n return x + 75\\n\\ndef step_76(x: int) -> int:\\n return x + 76\\n\\ndef step_77(x: int) -> int:\\n return x + 77\\n\\ndef step_78(x: int) -> int:\\n return x + 78\\n\\ndef step_79(x: int) -> int:\\n return x + 79\\n\"}" } ], "exception": null }, "rand:3:8": { "content": "\n\n", "reasoning_content": "Write the module.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"steps.py\", \"content\": \"\\\"\\\"\\\"Generated module used as a long tool argument.\\\"\\\"\\\"\\n\\nfrom __future__ import annotations\\n\\ndef step_00(x: int) -> int:\\n return x + 0\\n\\ndef step_01(x: int) -> int:\\n return x + 1\\n\\ndef step_02(x: int) -> int:\\n return x + 2\\n\\ndef step_03(x: int) -> int:\\n return x + 3\\n\\ndef step_04(x: int) -> int:\\n return x + 4\\n\\ndef step_05(x: int) -> int:\\n return x + 5\\n\\ndef step_06(x: int) -> int:\\n return x + 6\\n\\ndef step_07(x: int) -> int:\\n return x + 7\\n\\ndef step_08(x: int) -> int:\\n return x + 8\\n\\ndef step_09(x: int) -> int:\\n return x + 9\\n\\ndef step_10(x: int) -> int:\\n return x + 10\\n\\ndef step_11(x: int) -> int:\\n return x + 11\\n\\ndef step_12(x: int) -> int:\\n return x + 12\\n\\ndef step_13(x: int) -> int:\\n return x + 13\\n\\ndef step_14(x: int) -> int:\\n return x + 14\\n\\ndef step_15(x: int) -> int:\\n return x + 15\\n\\ndef step_16(x: int) -> int:\\n return x + 16\\n\\ndef step_17(x: int) -> int:\\n return x + 17\\n\\ndef step_18(x: int) -> int:\\n return x + 18\\n\\ndef step_19(x: int) -> int:\\n return x + 19\\n\\ndef step_20(x: int) -> int:\\n return x + 20\\n\\ndef step_21(x: int) -> int:\\n return x + 21\\n\\ndef step_22(x: int) -> int:\\n return x + 22\\n\\ndef step_23(x: int) -> int:\\n return x + 23\\n\\ndef step_24(x: int) -> int:\\n return x + 24\\n\\ndef step_25(x: int) -> int:\\n return x + 25\\n\\ndef step_26(x: int) -> int:\\n return x + 26\\n\\ndef step_27(x: int) -> int:\\n return x + 27\\n\\ndef step_28(x: int) -> int:\\n return x + 28\\n\\ndef step_29(x: int) -> int:\\n return x + 29\\n\\ndef step_30(x: int) -> int:\\n return x + 30\\n\\ndef step_31(x: int) -> int:\\n return x + 31\\n\\ndef step_32(x: int) -> int:\\n return x + 32\\n\\ndef step_33(x: int) -> int:\\n return x + 33\\n\\ndef step_34(x: int) -> int:\\n return x + 34\\n\\ndef step_35(x: int) -> int:\\n return x + 35\\n\\ndef step_36(x: int) -> int:\\n return x + 36\\n\\ndef step_37(x: int) -> int:\\n return x + 37\\n\\ndef step_38(x: int) -> int:\\n return x + 38\\n\\ndef step_39(x: int) -> int:\\n return x + 39\\n\\ndef step_40(x: int) -> int:\\n return x + 40\\n\\ndef step_41(x: int) -> int:\\n return x + 41\\n\\ndef step_42(x: int) -> int:\\n return x + 42\\n\\ndef step_43(x: int) -> int:\\n return x + 43\\n\\ndef step_44(x: int) -> int:\\n return x + 44\\n\\ndef step_45(x: int) -> int:\\n return x + 45\\n\\ndef step_46(x: int) -> int:\\n return x + 46\\n\\ndef step_47(x: int) -> int:\\n return x + 47\\n\\ndef step_48(x: int) -> int:\\n return x + 48\\n\\ndef step_49(x: int) -> int:\\n return x + 49\\n\\ndef step_50(x: int) -> int:\\n return x + 50\\n\\ndef step_51(x: int) -> int:\\n return x + 51\\n\\ndef step_52(x: int) -> int:\\n return x + 52\\n\\ndef step_53(x: int) -> int:\\n return x + 53\\n\\ndef step_54(x: int) -> int:\\n return x + 54\\n\\ndef step_55(x: int) -> int:\\n return x + 55\\n\\ndef step_56(x: int) -> int:\\n return x + 56\\n\\ndef step_57(x: int) -> int:\\n return x + 57\\n\\ndef step_58(x: int) -> int:\\n return x + 58\\n\\ndef step_59(x: int) -> int:\\n return x + 59\\n\\ndef step_60(x: int) -> int:\\n return x + 60\\n\\ndef step_61(x: int) -> int:\\n return x + 61\\n\\ndef step_62(x: int) -> int:\\n return x + 62\\n\\ndef step_63(x: int) -> int:\\n return x + 63\\n\\ndef step_64(x: int) -> int:\\n return x + 64\\n\\ndef step_65(x: int) -> int:\\n return x + 65\\n\\ndef step_66(x: int) -> int:\\n return x + 66\\n\\ndef step_67(x: int) -> int:\\n return x + 67\\n\\ndef step_68(x: int) -> int:\\n return x + 68\\n\\ndef step_69(x: int) -> int:\\n return x + 69\\n\\ndef step_70(x: int) -> int:\\n return x + 70\\n\\ndef step_71(x: int) -> int:\\n return x + 71\\n\\ndef step_72(x: int) -> int:\\n return x + 72\\n\\ndef step_73(x: int) -> int:\\n return x + 73\\n\\ndef step_74(x: int) -> int:\\n return x + 74\\n\\ndef step_75(x: int) -> int:\\n return x + 75\\n\\ndef step_76(x: int) -> int:\\n return x + 76\\n\\ndef step_77(x: int) -> int:\\n return x + 77\\n\\ndef step_78(x: int) -> int:\\n return x + 78\\n\\ndef step_79(x: int) -> int:\\n return x + 79\\n\"}" } ], "exception": null }, "rand:4:8": { "content": "\n\n", "reasoning_content": "Write the module.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"steps.py\", \"content\": \"\\\"\\\"\\\"Generated module used as a long tool argument.\\\"\\\"\\\"\\n\\nfrom __future__ import annotations\\n\\ndef step_00(x: int) -> int:\\n return x + 0\\n\\ndef step_01(x: int) -> int:\\n return x + 1\\n\\ndef step_02(x: int) -> int:\\n return x + 2\\n\\ndef step_03(x: int) -> int:\\n return x + 3\\n\\ndef step_04(x: int) -> int:\\n return x + 4\\n\\ndef step_05(x: int) -> int:\\n return x + 5\\n\\ndef step_06(x: int) -> int:\\n return x + 6\\n\\ndef step_07(x: int) -> int:\\n return x + 7\\n\\ndef step_08(x: int) -> int:\\n return x + 8\\n\\ndef step_09(x: int) -> int:\\n return x + 9\\n\\ndef step_10(x: int) -> int:\\n return x + 10\\n\\ndef step_11(x: int) -> int:\\n return x + 11\\n\\ndef step_12(x: int) -> int:\\n return x + 12\\n\\ndef step_13(x: int) -> int:\\n return x + 13\\n\\ndef step_14(x: int) -> int:\\n return x + 14\\n\\ndef step_15(x: int) -> int:\\n return x + 15\\n\\ndef step_16(x: int) -> int:\\n return x + 16\\n\\ndef step_17(x: int) -> int:\\n return x + 17\\n\\ndef step_18(x: int) -> int:\\n return x + 18\\n\\ndef step_19(x: int) -> int:\\n return x + 19\\n\\ndef step_20(x: int) -> int:\\n return x + 20\\n\\ndef step_21(x: int) -> int:\\n return x + 21\\n\\ndef step_22(x: int) -> int:\\n return x + 22\\n\\ndef step_23(x: int) -> int:\\n return x + 23\\n\\ndef step_24(x: int) -> int:\\n return x + 24\\n\\ndef step_25(x: int) -> int:\\n return x + 25\\n\\ndef step_26(x: int) -> int:\\n return x + 26\\n\\ndef step_27(x: int) -> int:\\n return x + 27\\n\\ndef step_28(x: int) -> int:\\n return x + 28\\n\\ndef step_29(x: int) -> int:\\n return x + 29\\n\\ndef step_30(x: int) -> int:\\n return x + 30\\n\\ndef step_31(x: int) -> int:\\n return x + 31\\n\\ndef step_32(x: int) -> int:\\n return x + 32\\n\\ndef step_33(x: int) -> int:\\n return x + 33\\n\\ndef step_34(x: int) -> int:\\n return x + 34\\n\\ndef step_35(x: int) -> int:\\n return x + 35\\n\\ndef step_36(x: int) -> int:\\n return x + 36\\n\\ndef step_37(x: int) -> int:\\n return x + 37\\n\\ndef step_38(x: int) -> int:\\n return x + 38\\n\\ndef step_39(x: int) -> int:\\n return x + 39\\n\\ndef step_40(x: int) -> int:\\n return x + 40\\n\\ndef step_41(x: int) -> int:\\n return x + 41\\n\\ndef step_42(x: int) -> int:\\n return x + 42\\n\\ndef step_43(x: int) -> int:\\n return x + 43\\n\\ndef step_44(x: int) -> int:\\n return x + 44\\n\\ndef step_45(x: int) -> int:\\n return x + 45\\n\\ndef step_46(x: int) -> int:\\n return x + 46\\n\\ndef step_47(x: int) -> int:\\n return x + 47\\n\\ndef step_48(x: int) -> int:\\n return x + 48\\n\\ndef step_49(x: int) -> int:\\n return x + 49\\n\\ndef step_50(x: int) -> int:\\n return x + 50\\n\\ndef step_51(x: int) -> int:\\n return x + 51\\n\\ndef step_52(x: int) -> int:\\n return x + 52\\n\\ndef step_53(x: int) -> int:\\n return x + 53\\n\\ndef step_54(x: int) -> int:\\n return x + 54\\n\\ndef step_55(x: int) -> int:\\n return x + 55\\n\\ndef step_56(x: int) -> int:\\n return x + 56\\n\\ndef step_57(x: int) -> int:\\n return x + 57\\n\\ndef step_58(x: int) -> int:\\n return x + 58\\n\\ndef step_59(x: int) -> int:\\n return x + 59\\n\\ndef step_60(x: int) -> int:\\n return x + 60\\n\\ndef step_61(x: int) -> int:\\n return x + 61\\n\\ndef step_62(x: int) -> int:\\n return x + 62\\n\\ndef step_63(x: int) -> int:\\n return x + 63\\n\\ndef step_64(x: int) -> int:\\n return x + 64\\n\\ndef step_65(x: int) -> int:\\n return x + 65\\n\\ndef step_66(x: int) -> int:\\n return x + 66\\n\\ndef step_67(x: int) -> int:\\n return x + 67\\n\\ndef step_68(x: int) -> int:\\n return x + 68\\n\\ndef step_69(x: int) -> int:\\n return x + 69\\n\\ndef step_70(x: int) -> int:\\n return x + 70\\n\\ndef step_71(x: int) -> int:\\n return x + 71\\n\\ndef step_72(x: int) -> int:\\n return x + 72\\n\\ndef step_73(x: int) -> int:\\n return x + 73\\n\\ndef step_74(x: int) -> int:\\n return x + 74\\n\\ndef step_75(x: int) -> int:\\n return x + 75\\n\\ndef step_76(x: int) -> int:\\n return x + 76\\n\\ndef step_77(x: int) -> int:\\n return x + 77\\n\\ndef step_78(x: int) -> int:\\n return x + 78\\n\\ndef step_79(x: int) -> int:\\n return x + 79\\n\"}" } ], "exception": null }, "rand:5:8": { "content": "\n\n", "reasoning_content": "Write the module.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"steps.py\", \"content\": \"\\\"\\\"\\\"Generated module used as a long tool argument.\\\"\\\"\\\"\\n\\nfrom __future__ import annotations\\n\\ndef step_00(x: int) -> int:\\n return x + 0\\n\\ndef step_01(x: int) -> int:\\n return x + 1\\n\\ndef step_02(x: int) -> int:\\n return x + 2\\n\\ndef step_03(x: int) -> int:\\n return x + 3\\n\\ndef step_04(x: int) -> int:\\n return x + 4\\n\\ndef step_05(x: int) -> int:\\n return x + 5\\n\\ndef step_06(x: int) -> int:\\n return x + 6\\n\\ndef step_07(x: int) -> int:\\n return x + 7\\n\\ndef step_08(x: int) -> int:\\n return x + 8\\n\\ndef step_09(x: int) -> int:\\n return x + 9\\n\\ndef step_10(x: int) -> int:\\n return x + 10\\n\\ndef step_11(x: int) -> int:\\n return x + 11\\n\\ndef step_12(x: int) -> int:\\n return x + 12\\n\\ndef step_13(x: int) -> int:\\n return x + 13\\n\\ndef step_14(x: int) -> int:\\n return x + 14\\n\\ndef step_15(x: int) -> int:\\n return x + 15\\n\\ndef step_16(x: int) -> int:\\n return x + 16\\n\\ndef step_17(x: int) -> int:\\n return x + 17\\n\\ndef step_18(x: int) -> int:\\n return x + 18\\n\\ndef step_19(x: int) -> int:\\n return x + 19\\n\\ndef step_20(x: int) -> int:\\n return x + 20\\n\\ndef step_21(x: int) -> int:\\n return x + 21\\n\\ndef step_22(x: int) -> int:\\n return x + 22\\n\\ndef step_23(x: int) -> int:\\n return x + 23\\n\\ndef step_24(x: int) -> int:\\n return x + 24\\n\\ndef step_25(x: int) -> int:\\n return x + 25\\n\\ndef step_26(x: int) -> int:\\n return x + 26\\n\\ndef step_27(x: int) -> int:\\n return x + 27\\n\\ndef step_28(x: int) -> int:\\n return x + 28\\n\\ndef step_29(x: int) -> int:\\n return x + 29\\n\\ndef step_30(x: int) -> int:\\n return x + 30\\n\\ndef step_31(x: int) -> int:\\n return x + 31\\n\\ndef step_32(x: int) -> int:\\n return x + 32\\n\\ndef step_33(x: int) -> int:\\n return x + 33\\n\\ndef step_34(x: int) -> int:\\n return x + 34\\n\\ndef step_35(x: int) -> int:\\n return x + 35\\n\\ndef step_36(x: int) -> int:\\n return x + 36\\n\\ndef step_37(x: int) -> int:\\n return x + 37\\n\\ndef step_38(x: int) -> int:\\n return x + 38\\n\\ndef step_39(x: int) -> int:\\n return x + 39\\n\\ndef step_40(x: int) -> int:\\n return x + 40\\n\\ndef step_41(x: int) -> int:\\n return x + 41\\n\\ndef step_42(x: int) -> int:\\n return x + 42\\n\\ndef step_43(x: int) -> int:\\n return x + 43\\n\\ndef step_44(x: int) -> int:\\n return x + 44\\n\\ndef step_45(x: int) -> int:\\n return x + 45\\n\\ndef step_46(x: int) -> int:\\n return x + 46\\n\\ndef step_47(x: int) -> int:\\n return x + 47\\n\\ndef step_48(x: int) -> int:\\n return x + 48\\n\\ndef step_49(x: int) -> int:\\n return x + 49\\n\\ndef step_50(x: int) -> int:\\n return x + 50\\n\\ndef step_51(x: int) -> int:\\n return x + 51\\n\\ndef step_52(x: int) -> int:\\n return x + 52\\n\\ndef step_53(x: int) -> int:\\n return x + 53\\n\\ndef step_54(x: int) -> int:\\n return x + 54\\n\\ndef step_55(x: int) -> int:\\n return x + 55\\n\\ndef step_56(x: int) -> int:\\n return x + 56\\n\\ndef step_57(x: int) -> int:\\n return x + 57\\n\\ndef step_58(x: int) -> int:\\n return x + 58\\n\\ndef step_59(x: int) -> int:\\n return x + 59\\n\\ndef step_60(x: int) -> int:\\n return x + 60\\n\\ndef step_61(x: int) -> int:\\n return x + 61\\n\\ndef step_62(x: int) -> int:\\n return x + 62\\n\\ndef step_63(x: int) -> int:\\n return x + 63\\n\\ndef step_64(x: int) -> int:\\n return x + 64\\n\\ndef step_65(x: int) -> int:\\n return x + 65\\n\\ndef step_66(x: int) -> int:\\n return x + 66\\n\\ndef step_67(x: int) -> int:\\n return x + 67\\n\\ndef step_68(x: int) -> int:\\n return x + 68\\n\\ndef step_69(x: int) -> int:\\n return x + 69\\n\\ndef step_70(x: int) -> int:\\n return x + 70\\n\\ndef step_71(x: int) -> int:\\n return x + 71\\n\\ndef step_72(x: int) -> int:\\n return x + 72\\n\\ndef step_73(x: int) -> int:\\n return x + 73\\n\\ndef step_74(x: int) -> int:\\n return x + 74\\n\\ndef step_75(x: int) -> int:\\n return x + 75\\n\\ndef step_76(x: int) -> int:\\n return x + 76\\n\\ndef step_77(x: int) -> int:\\n return x + 77\\n\\ndef step_78(x: int) -> int:\\n return x + 78\\n\\ndef step_79(x: int) -> int:\\n return x + 79\\n\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/q38-empty-reasoning", "family": "qwen3-xml", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.8-27B", "tokenizer": { "repo": "Qwen/Qwen3.8-27B", "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 248045, 74455, 198, 248068, 198 ] }, "stop": { "token": "<|im_end|>", "id": 248046, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "\n\n", "reasoning_content": "\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "streams": { "one": { "content": "\n\n", "reasoning_content": "\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": "\n\n", "reasoning_content": "\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": "\n\n", "reasoning_content": "\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:1:8": { "content": "\n\n", "reasoning_content": "\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": "\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:3:8": { "content": "\n\n", "reasoning_content": "\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:4:8": { "content": "\n\n", "reasoning_content": "\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:5:8": { "content": "\n\n", "reasoning_content": "\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/q38-thinking-disabled", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.8-27B", "tokenizer": { "repo": "Qwen/Qwen3.8-27B", "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 248068, 271, 248069, 271 ] }, "stop": { "token": "<|im_end|>", "id": 248046, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "qwen3-xml/q38-thinking-disabled-parallel", "family": "qwen3-xml", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.8-27B", "tokenizer": { "repo": "Qwen/Qwen3.8-27B", "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 248068, 271, 248069, 271 ] }, "stop": { "token": "<|im_end|>", "id": 248046, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\"}" }, { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "streams": { "one": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\"}" }, { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\"}" }, { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\"}" }, { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:1:8": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\"}" }, { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:2:8": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\"}" }, { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:3:8": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\"}" }, { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:4:8": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\"}" }, { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:5:8": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Oslo\"}" }, { "name": "get_time", "arguments_raw": "{}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/q38-thinking-disabled-no-call", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.8-27B", "tokenizer": { "repo": "Qwen/Qwen3.8-27B", "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": false, "chat_template_kwargs": { "enable_thinking": false }, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 198, 248068, 271, 248069, 271 ] }, "stop": { "token": "<|im_end|>", "id": 248046, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "qwen3-xml/q38-truncated-in-reasoning", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.8-27B", "tokenizer": { "repo": "Qwen/Qwen3.8-27B", "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 248045, 74455, 198, 248068, 198 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "qwen3-xml/q38-truncated-in-parameter", "family": "qwen3-xml", "status": "fail", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "fail", "detail": "returned 1 tool call(s) ['write_file'] for Output stopped (max_tokens) inside a parameter value, before and . The call is incomplete; a truncated 'rm -rf /tmp/build' must not become a complete 'rm -rf /' (vLLM #57699)." }, { "check": "expected_error", "strategy": "one", "status": "fail", "detail": "returned 1 tool call(s) ['write_file'] for Output stopped (max_tokens) inside a parameter value, before and . The call is incomplete; a truncated 'rm -rf /tmp/build' must not become a complete 'rm -rf /' (vLLM #57699)." }, { "check": "expected_error", "strategy": "special", "status": "fail", "detail": "returned 1 tool call(s) ['write_file'] for Output stopped (max_tokens) inside a parameter value, before and . The call is incomplete; a truncated 'rm -rf /tmp/build' must not become a complete 'rm -rf /' (vLLM #57699)." }, { "check": "expected_error", "strategy": "token", "status": "fail", "detail": "returned 1 tool call(s) ['write_file'] for Output stopped (max_tokens) inside a parameter value, before and . The call is incomplete; a truncated 'rm -rf /tmp/build' must not become a complete 'rm -rf /' (vLLM #57699)." }, { "check": "expected_error", "strategy": "rand:1:8", "status": "fail", "detail": "returned 1 tool call(s) ['write_file'] for Output stopped (max_tokens) inside a parameter value, before and . The call is incomplete; a truncated 'rm -rf /tmp/build' must not become a complete 'rm -rf /' (vLLM #57699)." }, { "check": "expected_error", "strategy": "rand:2:8", "status": "fail", "detail": "returned 1 tool call(s) ['write_file'] for Output stopped (max_tokens) inside a parameter value, before and . The call is incomplete; a truncated 'rm -rf /tmp/build' must not become a complete 'rm -rf /' (vLLM #57699)." }, { "check": "expected_error", "strategy": "rand:3:8", "status": "fail", "detail": "returned 1 tool call(s) ['write_file'] for Output stopped (max_tokens) inside a parameter value, before and . The call is incomplete; a truncated 'rm -rf /tmp/build' must not become a complete 'rm -rf /' (vLLM #57699)." }, { "check": "expected_error", "strategy": "rand:4:8", "status": "fail", "detail": "returned 1 tool call(s) ['write_file'] for Output stopped (max_tokens) inside a parameter value, before and . The call is incomplete; a truncated 'rm -rf /tmp/build' must not become a complete 'rm -rf /' (vLLM #57699)." }, { "check": "expected_error", "strategy": "rand:5:8", "status": "fail", "detail": "returned 1 tool call(s) ['write_file'] for Output stopped (max_tokens) inside a parameter value, before and . The call is incomplete; a truncated 'rm -rf /tmp/build' must not become a complete 'rm -rf /' (vLLM #57699)." }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"path\": \"/tmp/build.sh\", \"content\": \"rm -rf /\"}', got '{\"path\": \"/tmp/build.sh\"'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"path\": \"/tmp/build.sh\", \"content\": \"rm -rf /\"}', got '{\"path\": \"/tmp/build.sh\"'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"path\": \"/tmp/build.sh\", \"content\": \"rm -rf /\"}', got '{\"path\": \"/tmp/build.sh\"'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"path\": \"/tmp/build.sh\", \"content\": \"rm -rf /\"}', got '{\"path\": \"/tmp/build.sh\"'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"path\": \"/tmp/build.sh\", \"content\": \"rm -rf /\"}', got '{\"path\": \"/tmp/build.sh\"'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"path\": \"/tmp/build.sh\", \"content\": \"rm -rf /\"}', got '{\"path\": \"/tmp/build.sh\"'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"path\": \"/tmp/build.sh\", \"content\": \"rm -rf /\"}', got '{\"path\": \"/tmp/build.sh\"'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls[0].arguments: nonstream '{\"path\": \"/tmp/build.sh\", \"content\": \"rm -rf /\"}', got '{\"path\": \"/tmp/build.sh\"'" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"/tmp/build.sh\"': Expecting ',' delimiter: line 1 column 25 (char 24))" }, { "check": "arguments_json", "strategy": "special", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"/tmp/build.sh\"': Expecting ',' delimiter: line 1 column 25 (char 24))" }, { "check": "arguments_json", "strategy": "token", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"/tmp/build.sh\"': Expecting ',' delimiter: line 1 column 25 (char 24))" }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"/tmp/build.sh\"': Expecting ',' delimiter: line 1 column 25 (char 24))" }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"/tmp/build.sh\"': Expecting ',' delimiter: line 1 column 25 (char 24))" }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"/tmp/build.sh\"': Expecting ',' delimiter: line 1 column 25 (char 24))" }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"/tmp/build.sh\"': Expecting ',' delimiter: line 1 column 25 (char 24))" }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "fail", "detail": "[0] write_file: arguments are not valid JSON text ('{\"path\": \"/tmp/build.sh\"': Expecting ',' delimiter: line 1 column 25 (char 24))" }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "token", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "fail", "detail": "[0] write_file: arguments are not a JSON object; not validated" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.8-27B", "tokenizer": { "repo": "Qwen/Qwen3.8-27B", "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 248045, 74455, 198, 248068, 198 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "\n\n", "reasoning_content": "Write the script.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"/tmp/build.sh\", \"content\": \"rm -rf /\"}" } ], "exception": null }, "streams": { "one": { "content": "\n\n", "reasoning_content": "Write the script.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"/tmp/build.sh\"" } ], "exception": null }, "special": { "content": "\n\n", "reasoning_content": "Write the script.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"/tmp/build.sh\"" } ], "exception": null }, "token": { "content": "\n\n", "reasoning_content": "Write the script.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"/tmp/build.sh\"" } ], "exception": null }, "rand:1:8": { "content": "\n\n", "reasoning_content": "Write the script.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"/tmp/build.sh\"" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": "Write the script.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"/tmp/build.sh\"" } ], "exception": null }, "rand:3:8": { "content": "\n\n", "reasoning_content": "Write the script.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"/tmp/build.sh\"" } ], "exception": null }, "rand:4:8": { "content": "\n\n", "reasoning_content": "Write the script.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"/tmp/build.sh\"" } ], "exception": null }, "rand:5:8": { "content": "\n\n", "reasoning_content": "Write the script.\n", "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"/tmp/build.sh\"" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/q38-truncated-at-tool-call-open", "family": "qwen3-xml", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'The user wants the weather in Paris. I should call get_weather.', got 'The user wants the weather in Paris. I should call get_weather.\\n'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'The user wants the weather in Paris. I should call get_weather.', got 'The user wants the weather in Paris. I should call get_weather.\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'The user wants the weather in Paris. I should call get_weather.', got 'The user wants the weather in Paris. I should call get_weather.\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'The user wants the weather in Paris. I should call get_weather.', got 'The user wants the weather in Paris. I should call get_weather.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'The user wants the weather in Paris. I should call get_weather.', got 'The user wants the weather in Paris. I should call get_weather.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'The user wants the weather in Paris. I should call get_weather.', got 'The user wants the weather in Paris. I should call get_weather.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'The user wants the weather in Paris. I should call get_weather.', got 'The user wants the weather in Paris. I should call get_weather.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'The user wants the weather in Paris. I should call get_weather.', got 'The user wants the weather in Paris. I should call get_weather.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'The user wants the weather in Paris. I should call get_weather.', got 'The user wants the weather in Paris. I should call get_weather.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.8-27B", "tokenizer": { "repo": "Qwen/Qwen3.8-27B", "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 248045, 74455, 198, 248068, 198 ] }, "stop": { "token": null, "id": null, "rule": "truncated fixture: finish_reason length", "appended": false, "finish_reason": "length", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "\n\n", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.\n", "tool_calls": [], "exception": null }, "streams": { "one": { "content": "\n\n", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.\n", "tool_calls": [], "exception": null }, "special": { "content": "\n\n", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.\n", "tool_calls": [], "exception": null }, "token": { "content": "\n\n", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.\n", "tool_calls": [], "exception": null }, "rand:1:8": { "content": "\n\n", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.\n", "tool_calls": [], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.\n", "tool_calls": [], "exception": null }, "rand:3:8": { "content": "\n\n", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.\n", "tool_calls": [], "exception": null }, "rand:4:8": { "content": "\n\n", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.\n", "tool_calls": [], "exception": null }, "rand:5:8": { "content": "\n\n", "reasoning_content": "The user wants the weather in Paris. I should call get_weather.\n", "tool_calls": [], "exception": null } } } }, { "fixture_id": "qwen3-xml/q35-python-literals", "family": "qwen3-xml", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Set a repeating alarm.', got 'Set a repeating alarm.\\n'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Set a repeating alarm.', got 'Set a repeating alarm.\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Set a repeating alarm.', got 'Set a repeating alarm.\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Set a repeating alarm.', got 'Set a repeating alarm.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Set a repeating alarm.', got 'Set a repeating alarm.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Set a repeating alarm.', got 'Set a repeating alarm.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Set a repeating alarm.', got 'Set a repeating alarm.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Set a repeating alarm.', got 'Set a repeating alarm.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Set a repeating alarm.', got 'Set a repeating alarm.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.5-9B", "tokenizer": { "repo": "Qwen/Qwen3.5-9B", "revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 248045, 74455, 198, 248068, 198 ] }, "stop": { "token": "<|im_end|>", "id": 248046, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "\n\n", "reasoning_content": "Set a repeating alarm.\n", "tool_calls": [ { "name": "set_alarm", "arguments_raw": "{\"hour\": 6, \"minute\": 0, \"repeat\": true, \"label\": \"gym\"}" } ], "exception": null }, "streams": { "one": { "content": "\n\n", "reasoning_content": "Set a repeating alarm.\n", "tool_calls": [ { "name": "set_alarm", "arguments_raw": "{\"hour\": 6, \"minute\": 0, \"repeat\": true, \"label\": \"gym\"}" } ], "exception": null }, "special": { "content": "\n\n", "reasoning_content": "Set a repeating alarm.\n", "tool_calls": [ { "name": "set_alarm", "arguments_raw": "{\"hour\": 6, \"minute\": 0, \"repeat\": true, \"label\": \"gym\"}" } ], "exception": null }, "token": { "content": "\n\n", "reasoning_content": "Set a repeating alarm.\n", "tool_calls": [ { "name": "set_alarm", "arguments_raw": "{\"hour\": 6, \"minute\": 0, \"repeat\": true, \"label\": \"gym\"}" } ], "exception": null }, "rand:1:8": { "content": "\n\n", "reasoning_content": "Set a repeating alarm.\n", "tool_calls": [ { "name": "set_alarm", "arguments_raw": "{\"hour\": 6, \"minute\": 0, \"repeat\": true, \"label\": \"gym\"}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": "Set a repeating alarm.\n", "tool_calls": [ { "name": "set_alarm", "arguments_raw": "{\"hour\": 6, \"minute\": 0, \"repeat\": true, \"label\": \"gym\"}" } ], "exception": null }, "rand:3:8": { "content": "\n\n", "reasoning_content": "Set a repeating alarm.\n", "tool_calls": [ { "name": "set_alarm", "arguments_raw": "{\"hour\": 6, \"minute\": 0, \"repeat\": true, \"label\": \"gym\"}" } ], "exception": null }, "rand:4:8": { "content": "\n\n", "reasoning_content": "Set a repeating alarm.\n", "tool_calls": [ { "name": "set_alarm", "arguments_raw": "{\"hour\": 6, \"minute\": 0, \"repeat\": true, \"label\": \"gym\"}" } ], "exception": null }, "rand:5:8": { "content": "\n\n", "reasoning_content": "Set a repeating alarm.\n", "tool_calls": [ { "name": "set_alarm", "arguments_raw": "{\"hour\": 6, \"minute\": 0, \"repeat\": true, \"label\": \"gym\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/q35-single-call", "family": "qwen3-xml", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Weather lookup.', got 'Weather lookup.\\n'" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Weather lookup.', got 'Weather lookup.\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Weather lookup.', got 'Weather lookup.\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Weather lookup.', got 'Weather lookup.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Weather lookup.', got 'Weather lookup.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Weather lookup.', got 'Weather lookup.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Weather lookup.', got 'Weather lookup.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Weather lookup.', got 'Weather lookup.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n\\n'; reasoning_content (whitespace only): expected 'Weather lookup.', got 'Weather lookup.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": "Qwen3Detector", "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3" }, "model": "Qwen/Qwen3.5-9B", "tokenizer": { "repo": "Qwen/Qwen3.5-9B", "revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "model_type": "qwen3_5" }, "hf_config_error": null, "chat_template_sha256": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715", "template_force_reasoning": false, "template_reasoning_config": "ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None, effort_kwarg=None)", "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": true, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 248045, 74455, 198, 248068, 198 ] }, "stop": { "token": "<|im_end|>", "id": 248046, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "\n\n", "reasoning_content": "Weather lookup.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" } ], "exception": null }, "streams": { "one": { "content": "\n\n", "reasoning_content": "Weather lookup.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" } ], "exception": null }, "special": { "content": "\n\n", "reasoning_content": "Weather lookup.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" } ], "exception": null }, "token": { "content": "\n\n", "reasoning_content": "Weather lookup.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" } ], "exception": null }, "rand:1:8": { "content": "\n\n", "reasoning_content": "Weather lookup.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" } ], "exception": null }, "rand:2:8": { "content": "\n\n", "reasoning_content": "Weather lookup.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" } ], "exception": null }, "rand:3:8": { "content": "\n\n", "reasoning_content": "Weather lookup.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" } ], "exception": null }, "rand:4:8": { "content": "\n\n", "reasoning_content": "Weather lookup.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" } ], "exception": null }, "rand:5:8": { "content": "\n\n", "reasoning_content": "Weather lookup.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/coder-single-call", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": null, "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": null }, "model": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "tokenizer": { "repo": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3MoeForCausalLM" ], "model_type": "qwen3_moe" }, "hf_config_error": null, "chat_template_sha256": "5a38bfa05833266240066aedc497decc9b00cc0d3e3b8cceea98cf530196ab06", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|im_end|>", "id": 151645, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "qwen3-xml/coder-parallel-calls", "family": "qwen3-xml", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": null, "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": null }, "model": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "tokenizer": { "repo": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3MoeForCausalLM" ], "model_type": "qwen3_moe" }, "hf_config_error": null, "chat_template_sha256": "5a38bfa05833266240066aedc497decc9b00cc0d3e3b8cceea98cf530196ab06", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|im_end|>", "id": 151645, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "streams": { "one": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "special": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "token": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:1:8": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:2:8": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:3:8": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:4:8": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null }, "rand:5:8": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Zürich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"query\": \"café \\\"best\\\"\", \"filters\": {\"tags\": [\"a\", \"b\"], \"max\": 3}}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/coder-text-before-call", "family": "qwen3-xml", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "content (whitespace only): expected \"I'll create the file now.\", got \"I'll create the file now.\\n\\n\"" }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected \"I'll create the file now.\", got \"I'll create the file now.\\n\\n\"" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected \"I'll create the file now.\", got \"I'll create the file now.\\n\\n\"" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected \"I'll create the file now.\", got \"I'll create the file now.\\n\\n\"" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected \"I'll create the file now.\", got \"I'll create the file now.\\n\\n\"" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected \"I'll create the file now.\", got \"I'll create the file now.\\n\\n\"" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected \"I'll create the file now.\", got \"I'll create the file now.\\n\\n\"" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected \"I'll create the file now.\", got \"I'll create the file now.\\n\\n\"" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected \"I'll create the file now.\", got \"I'll create the file now.\\n\\n\"" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": null, "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": null }, "model": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "tokenizer": { "repo": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3MoeForCausalLM" ], "model_type": "qwen3_moe" }, "hf_config_error": null, "chat_template_sha256": "5a38bfa05833266240066aedc497decc9b00cc0d3e3b8cceea98cf530196ab06", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|im_end|>", "id": 151645, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": "I'll create the file now.\n\n", "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"hello.py\", \"content\": \"def greet(name: str) -> str:\\n \\\"\\\"\\\"Say hi.\\\"\\\"\\\"\\n return f\\\"Hello, {name}!\\\"\\n\\n\\nprint(greet(\\\"Zoë\\\"))\"}" } ], "exception": null }, "streams": { "one": { "content": "I'll create the file now.\n\n", "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"hello.py\", \"content\": \"def greet(name: str) -> str:\\n \\\"\\\"\\\"Say hi.\\\"\\\"\\\"\\n return f\\\"Hello, {name}!\\\"\\n\\n\\nprint(greet(\\\"Zoë\\\"))\"}" } ], "exception": null }, "special": { "content": "I'll create the file now.\n\n", "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"hello.py\", \"content\": \"def greet(name: str) -> str:\\n \\\"\\\"\\\"Say hi.\\\"\\\"\\\"\\n return f\\\"Hello, {name}!\\\"\\n\\n\\nprint(greet(\\\"Zoë\\\"))\"}" } ], "exception": null }, "token": { "content": "I'll create the file now.\n\n", "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"hello.py\", \"content\": \"def greet(name: str) -> str:\\n \\\"\\\"\\\"Say hi.\\\"\\\"\\\"\\n return f\\\"Hello, {name}!\\\"\\n\\n\\nprint(greet(\\\"Zoë\\\"))\"}" } ], "exception": null }, "rand:1:8": { "content": "I'll create the file now.\n\n", "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"hello.py\", \"content\": \"def greet(name: str) -> str:\\n \\\"\\\"\\\"Say hi.\\\"\\\"\\\"\\n return f\\\"Hello, {name}!\\\"\\n\\n\\nprint(greet(\\\"Zoë\\\"))\"}" } ], "exception": null }, "rand:2:8": { "content": "I'll create the file now.\n\n", "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"hello.py\", \"content\": \"def greet(name: str) -> str:\\n \\\"\\\"\\\"Say hi.\\\"\\\"\\\"\\n return f\\\"Hello, {name}!\\\"\\n\\n\\nprint(greet(\\\"Zoë\\\"))\"}" } ], "exception": null }, "rand:3:8": { "content": "I'll create the file now.\n\n", "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"hello.py\", \"content\": \"def greet(name: str) -> str:\\n \\\"\\\"\\\"Say hi.\\\"\\\"\\\"\\n return f\\\"Hello, {name}!\\\"\\n\\n\\nprint(greet(\\\"Zoë\\\"))\"}" } ], "exception": null }, "rand:4:8": { "content": "I'll create the file now.\n\n", "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"hello.py\", \"content\": \"def greet(name: str) -> str:\\n \\\"\\\"\\\"Say hi.\\\"\\\"\\\"\\n return f\\\"Hello, {name}!\\\"\\n\\n\\nprint(greet(\\\"Zoë\\\"))\"}" } ], "exception": null }, "rand:5:8": { "content": "I'll create the file now.\n\n", "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"hello.py\", \"content\": \"def greet(name: str) -> str:\\n \\\"\\\"\\\"Say hi.\\\"\\\"\\\"\\n return f\\\"Hello, {name}!\\\"\\n\\n\\nprint(greet(\\\"Zoë\\\"))\"}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/coder-no-call", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": null, "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": null }, "model": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "tokenizer": { "repo": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3MoeForCausalLM" ], "model_type": "qwen3_moe" }, "hf_config_error": null, "chat_template_sha256": "5a38bfa05833266240066aedc497decc9b00cc0d3e3b8cceea98cf530196ab06", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|im_end|>", "id": 151645, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "qwen3-xml/coder-python-literals", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": null, "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": null }, "model": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "tokenizer": { "repo": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3MoeForCausalLM" ], "model_type": "qwen3_moe" }, "hf_config_error": null, "chat_template_sha256": "5a38bfa05833266240066aedc497decc9b00cc0d3e3b8cceea98cf530196ab06", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|im_end|>", "id": 151645, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "qwen3-xml/coder-empty-arguments", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": null, "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": null }, "model": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "tokenizer": { "repo": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3MoeForCausalLM" ], "model_type": "qwen3_moe" }, "hf_config_error": null, "chat_template_sha256": "5a38bfa05833266240066aedc497decc9b00cc0d3e3b8cceea98cf530196ab06", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|im_end|>", "id": 151645, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "qwen3-xml/coder-marker-in-arguments", "family": "qwen3-xml", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "tool_calls[0].arguments: expected '{\"path\": \"x.md\", \"content\": \"Close with then ; open with .\"}', got '{\"path\": \"x.md\", \"content\": \"Close with \"}'" }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "content: expected None, got '; open with .\\n\\n\\n'; tool_calls: expected ['write_file'], got ['write_file', 'x']" }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "content: expected None, got '; open with .\\n\\n\\n'; tool_calls: expected ['write_file'], got ['write_file', 'x']" }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "content: expected None, got '; open with .\\n\\n\\n'; tool_calls: expected ['write_file'], got ['write_file', 'x']" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "content: expected None, got '; open with .\\n\\n\\n'; tool_calls: expected ['write_file'], got ['write_file', 'x']" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "content: expected None, got '; open with .\\n\\n\\n'; tool_calls: expected ['write_file'], got ['write_file', 'x']" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "content: expected None, got '; open with .\\n\\n\\n'; tool_calls: expected ['write_file'], got ['write_file', 'x']" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "content: expected None, got '; open with .\\n\\n\\n'; tool_calls: expected ['write_file'], got ['write_file', 'x']" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "content: expected None, got '; open with .\\n\\n\\n'; tool_calls: expected ['write_file'], got ['write_file', 'x']" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream None, got '; open with .\\n\\n\\n'; tool_calls: nonstream ['write_file'], got ['write_file', 'x']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream None, got '; open with .\\n\\n\\n'; tool_calls: nonstream ['write_file'], got ['write_file', 'x']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream None, got '; open with .\\n\\n\\n'; tool_calls: nonstream ['write_file'], got ['write_file', 'x']" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream None, got '; open with .\\n\\n\\n'; tool_calls: nonstream ['write_file'], got ['write_file', 'x']" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream None, got '; open with .\\n\\n\\n'; tool_calls: nonstream ['write_file'], got ['write_file', 'x']" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream None, got '; open with .\\n\\n\\n'; tool_calls: nonstream ['write_file'], got ['write_file', 'x']" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream None, got '; open with .\\n\\n\\n'; tool_calls: nonstream ['write_file'], got ['write_file', 'x']" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream None, got '; open with .\\n\\n\\n'; tool_calls: nonstream ['write_file'], got ['write_file', 'x']" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "special", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "token", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "fail", "detail": "content contains ''" }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "fail", "detail": "content contains ''" }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "fail", "detail": "[1] 'x' is not an offered tool" }, { "check": "arguments_schema", "strategy": "special", "status": "fail", "detail": "[1] 'x' is not an offered tool" }, { "check": "arguments_schema", "strategy": "token", "status": "fail", "detail": "[1] 'x' is not an offered tool" }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "fail", "detail": "[1] 'x' is not an offered tool" }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "fail", "detail": "[1] 'x' is not an offered tool" }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "fail", "detail": "[1] 'x' is not an offered tool" }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "fail", "detail": "[1] 'x' is not an offered tool" }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "fail", "detail": "[1] 'x' is not an offered tool" } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": null, "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": null }, "model": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "tokenizer": { "repo": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "revision": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3MoeForCausalLM" ], "model_type": "qwen3_moe" }, "hf_config_error": null, "chat_template_sha256": "5a38bfa05833266240066aedc497decc9b00cc0d3e3b8cceea98cf530196ab06", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|im_end|>", "id": 151645, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"x.md\", \"content\": \"Close with \"}" } ], "exception": null }, "streams": { "one": { "content": "; open with .\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"x.md\", \"content\": \"Close with \"}" }, { "name": "x", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": "; open with .\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"x.md\", \"content\": \"Close with \"}" }, { "name": "x", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": "; open with .\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"x.md\", \"content\": \"Close with \"}" }, { "name": "x", "arguments_raw": "{}" } ], "exception": null }, "rand:1:8": { "content": "; open with .\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"x.md\", \"content\": \"Close with \"}" }, { "name": "x", "arguments_raw": "{}" } ], "exception": null }, "rand:2:8": { "content": "; open with .\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"x.md\", \"content\": \"Close with \"}" }, { "name": "x", "arguments_raw": "{}" } ], "exception": null }, "rand:3:8": { "content": "; open with .\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"x.md\", \"content\": \"Close with \"}" }, { "name": "x", "arguments_raw": "{}" } ], "exception": null }, "rand:4:8": { "content": "; open with .\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"x.md\", \"content\": \"Close with \"}" }, { "name": "x", "arguments_raw": "{}" } ], "exception": null }, "rand:5:8": { "content": "; open with .\n\n\n", "reasoning_content": null, "tool_calls": [ { "name": "write_file", "arguments_raw": "{\"path\": \"x.md\", \"content\": \"Close with \"}" }, { "name": "x", "arguments_raw": "{}" } ], "exception": null } } } }, { "fixture_id": "qwen3-xml/coder-next-json-literals", "family": "qwen3-xml", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": null, "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": null }, "model": "Qwen/Qwen3-Coder-Next", "tokenizer": { "repo": "Qwen/Qwen3-Coder-Next", "revision": "a7fbcb5c0e12d62a448eaa0e260346bf5dcc0feb", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3NextForCausalLM" ], "model_type": "qwen3_next" }, "hf_config_error": null, "chat_template_sha256": "c79a833039a43602150cce0902403d6e376c50930c1b2a139b2964e1f0c322a0", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|im_end|>", "id": 151645, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] } }, { "fixture_id": "qwen3-xml/coder-next-parallel-calls", "family": "qwen3-xml", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream None, got '\\n'" }, { "check": "split_invariance", "strategy": "*", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "engine": "sglang", "version": "0.5.20", "tool_call_parser": "qwen3_coder", "reasoning_parser": null, "tool_call_detector": "Qwen3CoderDetector", "reasoning_detector": null, "auto_detected": { "tool_call_parser": "qwen3_coder", "reasoning_parser": null }, "model": "Qwen/Qwen3-Coder-Next", "tokenizer": { "repo": "Qwen/Qwen3-Coder-Next", "revision": "a7fbcb5c0e12d62a448eaa0e260346bf5dcc0feb", "class": "Qwen2Tokenizer", "loader": "sglang.srt.utils.hf_transformers_utils.get_tokenizer(revision=, tokenizer_revision=)", "trust_remote_code": false }, "tokenizer_mode": "hf", "hf_config": { "architectures": [ "Qwen3NextForCausalLM" ], "model_type": "qwen3_next" }, "hf_config_error": null, "chat_template_sha256": "c79a833039a43602150cce0902403d6e376c50930c1b2a139b2964e1f0c322a0", "template_force_reasoning": false, "template_reasoning_config": null, "chat_encoding_spec": null, "thinking": null, "chat_template_kwargs": null, "reasoning_effort": null, "reasoning_enabled": false, "tools_offered": 7, "tool_choice": "auto", "separate_reasoning": true, "stream_reasoning": true, "skip_special_tokens": false, "spaces_between_special_tokens": true, "no_stop_trim": false, "detokenizer": "DetokenizerManager._decode_batch_token_id_output", "prompt_tail": { "source": "generation_prompt", "ids": [ 151644, 77091, 198 ] }, "stop": { "token": "<|im_end|>", "id": 151645, "rule": "first stop token of the reference model", "appended": true, "finish_reason": "stop", "kept_by_engine": false }, "notes": [] }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Lyon\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Nice\"}" } ], "exception": null }, "streams": { "one": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Lyon\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Nice\"}" } ], "exception": null }, "special": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Lyon\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Nice\"}" } ], "exception": null }, "token": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Lyon\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Nice\"}" } ], "exception": null }, "rand:1:8": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Lyon\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Nice\"}" } ], "exception": null }, "rand:2:8": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Lyon\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Nice\"}" } ], "exception": null }, "rand:3:8": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Lyon\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Nice\"}" } ], "exception": null }, "rand:4:8": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Lyon\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Nice\"}" } ], "exception": null }, "rand:5:8": { "content": "\n", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Lyon\"}" }, { "name": "get_weather", "arguments_raw": "{\"city\": \"Nice\"}" } ], "exception": null } } } } ], "summary": { "totals": { "pass": 189, "soft_pass": 62, "fail": 197, "error": 0, "unsupported": 21 }, "by_family": { "deepseek": { "pass": 13, "soft_pass": 23, "fail": 28, "error": 0, "unsupported": 21 }, "gemma4": { "pass": 30, "soft_pass": 5, "fail": 13, "error": 0, "unsupported": 0 }, "glm": { "pass": 25, "soft_pass": 2, "fail": 24, "error": 0, "unsupported": 0 }, "gpt-oss": { "pass": 19, "soft_pass": 0, "fail": 32, "error": 0, "unsupported": 0 }, "kimi": { "pass": 44, "soft_pass": 0, "fail": 5, "error": 0, "unsupported": 0 }, "llama": { "pass": 9, "soft_pass": 0, "fail": 25, "error": 0, "unsupported": 0 }, "mistral": { "pass": 23, "soft_pass": 1, "fail": 23, "error": 0, "unsupported": 0 }, "qwen3-hermes": { "pass": 7, "soft_pass": 4, "fail": 39, "error": 0, "unsupported": 0 }, "qwen3-xml": { "pass": 19, "soft_pass": 27, "fail": 8, "error": 0, "unsupported": 0 } } } }