{ "schema_version": "0.1", "canitoolcall_version": "0.1.0.dev0", "engine": { "name": "transformers", "version": "5.17.0", "commit": null, "details": { "tokenizers": "0.23.2", "huggingface_hub": "1.33.0", "jinja2": "3.1.6", "jmespath": "1.1.0", "torch": null, "pinned_repos": { "google/gemma-4-E2B-it": "3e22461f65e89153144f8adb70e3b8c2cc9845a7", "google/gemma-4-E4B-it": "ee0ef6023621cff504d758262d4e04895a5af4a2", "google/gemma-4-12B-it": "707f0a3b8a3c7ad586ed01e27eafbad8a27dd0f7", "google/gemma-4-26B-A4B-it": "4d7ae4984b7db7de8f8457170b3f1a419ee76d52", "google/gemma-4-31B-it": "842da3794eaa0b77d5f08bae87a17459d91ff475" }, "pinned_version": "5.17.0", "tokens_per_step": "one" } }, "run": { "started_at": "2026-09-26T05:18:44Z", "finished_at": "2026-09-26T05:18:47Z", "platform": "linux-x86_64", "python": "3.12.14", "fixtures_digest": "a1e7b55367e195e8aa6e2ef746b2cc141b44f37caad0887ff8f30b104616f836", "strategies": [ "one", "special", "token", "rand:1:8", "rand:2:8", "rand:3:8", "rand:4:8", "rand:5:8" ], "normalization": "soft-v1", "synthetic_strategies": [ "one", "special", "rand:1:8", "rand:2:8", "rand:3:8", "rand:4:8", "rand:5:8" ] }, "cases": [ { "fixture_id": "deepseek/vllm-v3-no-tool-calls", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/vllm-v3-single-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/vllm-v3-parallel-no-newline", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/vllm-v3-various-data-types", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/vllm-v3-escaped-strings", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/vllm-v31-text-before-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/vllm-v41-text-before-parallel-calls", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/vllm-v41-reasoning-text-before-parallel-calls", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/sglang-v32-text-before-empty-invoke", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/sglang-v4-self-closing-invoke", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/vllm-v3-malformed-missing-brace", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/vllm-v3-malformed-missing-call-tokens", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v3-single-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v3-parallel-two-calls", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v3-no-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v3-text-before-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v3-empty-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v3-numeric-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v3-unicode-emoji", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v3-marker-in-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v3-truncated-mid-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v3-nested-json", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v31-single-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v31-parallel-two-calls", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v31-no-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v31-text-before-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v31-empty-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v31-numeric-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v31-unicode-emoji", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v31-marker-in-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v31-truncated-mid-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v31-multi-turn-after-tool-result", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v31-nested-json", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v32-single-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v32-parallel-two-calls", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v32-no-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v32-text-before-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v32-empty-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v32-numeric-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v32-unicode-emoji", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v32-marker-in-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v32-truncated-mid-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v32-reasoning-single-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v32-reasoning-parallel-text-before", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v32-reasoning-no-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v32-string-false-json", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v32-unescaped-string-value", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v32-truncated-in-reasoning", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v32-truncated-second-parallel-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v4-single-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v4-parallel-two-calls", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v4-no-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v4-text-before-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v4-empty-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v4-numeric-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v4-unicode-emoji", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v4-marker-in-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v4-truncated-mid-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v4-multi-turn-after-tool-result", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v4-reasoning-single-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v4-reasoning-parallel-text-before", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v4-reasoning-no-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v4-string-false-json", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v4-unescaped-string-value", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v4-truncated-in-reasoning", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v4-truncated-second-parallel-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v41-single-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v41-parallel-two-calls", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v41-no-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v41-text-before-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v41-empty-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v41-numeric-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v41-unicode-emoji", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v41-marker-in-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v41-truncated-mid-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v41-multi-turn-after-tool-result", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v41-reasoning-single-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v41-reasoning-parallel-text-before", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v41-reasoning-no-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v41-string-false-json", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v41-unescaped-string-value", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v41-truncated-in-reasoning", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v41-truncated-second-parallel-call", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v41-long-arguments", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "deepseek/v41-whitespace-string-value", "family": "deepseek", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'deepseek' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gemma4/single-call", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null } }, { "fixture_id": "gemma4/single-call-thinking", "family": "gemma4", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream \"The user wants the current weather in Paris. I'll call get_weather.\", got \"The user wants the current weather in Paris. I'll call get_weather.\\n\"" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": true }, "stop_token_appended": null }, "observed": { "nonstream": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "token": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "The user wants the current weather in Paris. I'll call get_weather.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Paris\", \"unit\": \"c\"}" } ], "exception": null } } } }, { "fixture_id": "gemma4/no-call", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null } }, { "fixture_id": "gemma4/no-call-thinking", "family": "gemma4", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got 'The user is just greeting me. No tool is needed.\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got 'The user is just greeting me. No tool is needed.\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got 'The user is just greeting me. No tool is needed.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got 'The user is just greeting me. No tool is needed.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got 'The user is just greeting me. No tool is needed.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got 'The user is just greeting me. No tool is needed.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got 'The user is just greeting me. No tool is needed.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'The user is just greeting me. No tool is needed.', got 'The user is just greeting me. No tool is needed.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream 'The user is just greeting me. No tool is needed.', got 'The user is just greeting me. No tool is needed.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream 'The user is just greeting me. No tool is needed.', got 'The user is just greeting me. No tool is needed.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream 'The user is just greeting me. No tool is needed.', got 'The user is just greeting me. No tool is needed.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream 'The user is just greeting me. No tool is needed.', got 'The user is just greeting me. No tool is needed.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream 'The user is just greeting me. No tool is needed.', got 'The user is just greeting me. No tool is needed.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream 'The user is just greeting me. No tool is needed.', got 'The user is just greeting me. No tool is needed.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream 'The user is just greeting me. No tool is needed.', got 'The user is just greeting me. No tool is needed.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream 'The user is just greeting me. No tool is needed.', got 'The user is just greeting me. No tool is needed.\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": true }, "stop_token_appended": null }, "observed": { "nonstream": { "content": "Hello! How can I help you today?", "reasoning_content": "The user is just greeting me. No tool is needed.", "tool_calls": [], "exception": null }, "streams": { "one": { "content": "Hello! How can I help you today?", "reasoning_content": "The user is just greeting me. No tool is needed.\n", "tool_calls": [], "exception": null }, "special": { "content": "Hello! How can I help you today?", "reasoning_content": "The user is just greeting me. No tool is needed.\n", "tool_calls": [], "exception": null }, "token": { "content": "Hello! How can I help you today?", "reasoning_content": "The user is just greeting me. No tool is needed.\n", "tool_calls": [], "exception": null }, "rand:1:8": { "content": "Hello! How can I help you today?", "reasoning_content": "The user is just greeting me. No tool is needed.\n", "tool_calls": [], "exception": null }, "rand:2:8": { "content": "Hello! How can I help you today?", "reasoning_content": "The user is just greeting me. No tool is needed.\n", "tool_calls": [], "exception": null }, "rand:3:8": { "content": "Hello! How can I help you today?", "reasoning_content": "The user is just greeting me. No tool is needed.\n", "tool_calls": [], "exception": null }, "rand:4:8": { "content": "Hello! How can I help you today?", "reasoning_content": "The user is just greeting me. No tool is needed.\n", "tool_calls": [], "exception": null }, "rand:5:8": { "content": "Hello! How can I help you today?", "reasoning_content": "The user is just greeting me. No tool is needed.\n", "tool_calls": [], "exception": null } } } }, { "fixture_id": "gemma4/text-after-call", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null } }, { "fixture_id": "gemma4/text-after-call-thinking", "family": "gemma4", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I need live weather data for London.', got 'I need live weather data for London.\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I need live weather data for London.', got 'I need live weather data for London.\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I need live weather data for London.', got 'I need live weather data for London.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I need live weather data for London.', got 'I need live weather data for London.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I need live weather data for London.', got 'I need live weather data for London.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I need live weather data for London.', got 'I need live weather data for London.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I need live weather data for London.', got 'I need live weather data for London.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I need live weather data for London.', got 'I need live weather data for London.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream 'I need live weather data for London.', got 'I need live weather data for London.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream 'I need live weather data for London.', got 'I need live weather data for London.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream 'I need live weather data for London.', got 'I need live weather data for London.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream 'I need live weather data for London.', got 'I need live weather data for London.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream 'I need live weather data for London.', got 'I need live weather data for London.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream 'I need live weather data for London.', got 'I need live weather data for London.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream 'I need live weather data for London.', got 'I need live weather data for London.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream 'I need live weather data for London.', got 'I need live weather data for London.\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": true }, "stop_token_appended": null }, "observed": { "nonstream": { "content": "Let me check the current conditions in London.", "reasoning_content": "I need live weather data for London.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "streams": { "one": { "content": "Let me check the current conditions in London.", "reasoning_content": "I need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "special": { "content": "Let me check the current conditions in London.", "reasoning_content": "I need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "token": { "content": "Let me check the current conditions in London.", "reasoning_content": "I need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:1:8": { "content": "Let me check the current conditions in London.", "reasoning_content": "I need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:2:8": { "content": "Let me check the current conditions in London.", "reasoning_content": "I need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:3:8": { "content": "Let me check the current conditions in London.", "reasoning_content": "I need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:4:8": { "content": "Let me check the current conditions in London.", "reasoning_content": "I need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null }, "rand:5:8": { "content": "Let me check the current conditions in London.", "reasoning_content": "I need live weather data for London.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"London\"}" } ], "exception": null } } } }, { "fixture_id": "gemma4/empty-arguments", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null } }, { "fixture_id": "gemma4/empty-arguments-thinking", "family": "gemma4", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'get_time takes no arguments.', got 'get_time takes no arguments.\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'get_time takes no arguments.', got 'get_time takes no arguments.\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'get_time takes no arguments.', got 'get_time takes no arguments.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'get_time takes no arguments.', got 'get_time takes no arguments.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'get_time takes no arguments.', got 'get_time takes no arguments.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'get_time takes no arguments.', got 'get_time takes no arguments.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'get_time takes no arguments.', got 'get_time takes no arguments.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'get_time takes no arguments.', got 'get_time takes no arguments.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream 'get_time takes no arguments.', got 'get_time takes no arguments.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream 'get_time takes no arguments.', got 'get_time takes no arguments.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream 'get_time takes no arguments.', got 'get_time takes no arguments.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream 'get_time takes no arguments.', got 'get_time takes no arguments.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream 'get_time takes no arguments.', got 'get_time takes no arguments.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream 'get_time takes no arguments.', got 'get_time takes no arguments.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream 'get_time takes no arguments.', got 'get_time takes no arguments.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream 'get_time takes no arguments.', got 'get_time takes no arguments.\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": true }, "stop_token_appended": null }, "observed": { "nonstream": { "content": null, "reasoning_content": "get_time takes no arguments.", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "get_time takes no arguments.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "get_time takes no arguments.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "token": { "content": null, "reasoning_content": "get_time takes no arguments.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "get_time takes no arguments.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "get_time takes no arguments.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "get_time takes no arguments.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "get_time takes no arguments.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "get_time takes no arguments.\n", "tool_calls": [ { "name": "get_time", "arguments_raw": "{}" } ], "exception": null } } } }, { "fixture_id": "gemma4/bug-ollama-18390-key-with-spaces", "family": "gemma4", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{name:<|\"|>Demo<|\"|>, connections:{Basic LLM Chain:{main:[[{node:<|\"|>X<|\"|>, type:..." }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{name:<|\"|>Demo<|\"|>, connections:{Basic LLM Chain:{main:[[{node:<|\"|>X<|\"|>, type:..." }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{name:<|\"|>Demo<|\"|>, connections:{Basic LLM Chain:{main:[[{node:<|\"|>X<|\"|>, type:..." }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{name:<|\"|>Demo<|\"|>, connections:{Basic LLM Chain:{main:[[{node:<|\"|>X<|\"|>, type:..." }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{name:<|\"|>Demo<|\"|>, connections:{Basic LLM Chain:{main:[[{node:<|\"|>X<|\"|>, type:..." }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{name:<|\"|>Demo<|\"|>, connections:{Basic LLM Chain:{main:[[{node:<|\"|>X<|\"|>, type:..." }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{name:<|\"|>Demo<|\"|>, connections:{Basic LLM Chain:{main:[[{node:<|\"|>X<|\"|>, type:..." }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{name:<|\"|>Demo<|\"|>, connections:{Basic LLM Chain:{main:[[{node:<|\"|>X<|\"|>, type:..." }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{name:<|\"|>Demo<|\"|>, connections:{Basic LLM Chain:{main:[[{node:<|\"|>X<|\"|>, type:..." }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{name:<|\"|>Demo<|\"|>, connections:{Basic LLM Chain:{main:[[{node:<|\"|>X<|\"|>, type:<|\"|>main<|\"|>, index:0}]]}}}'\nTransformed: '{\"name\":\"Demo\", connections:{Basic LLM Chain:{\"main\":[[{\"node\":\"X\", type:\"main\", index:0}]]}}}'\nError: Expecting property name enclosed in double quotes: line 1 column 17 (char 16)" }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{name:<|\"|>Demo<|\"|>, connections:{Basic LLM Chain:{main:[[{node:<|\"|>X<|\"|>, type:<|\"|>main<|\"|>, index:0}]]}}}'\nTransformed: '{\"name\":\"Demo\", connections:{Basic LLM Chain:{\"main\":[[{\"node\":\"X\", type:\"main\", index:0}]]}}}'\nError: Expecting property name enclosed in double quotes: line 1 column 17 (char 16)" }, "special": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{name:<|\"|>Demo<|\"|>, connections:{Basic LLM Chain:{main:[[{node:<|\"|>X<|\"|>, type:<|\"|>main<|\"|>, index:0}]]}}}'\nTransformed: '{\"name\":\"Demo\", connections:{Basic LLM Chain:{\"main\":[[{\"node\":\"X\", type:\"main\", index:0}]]}}}'\nError: Expecting property name enclosed in double quotes: line 1 column 17 (char 16)" }, "token": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{name:<|\"|>Demo<|\"|>, connections:{Basic LLM Chain:{main:[[{node:<|\"|>X<|\"|>, type:<|\"|>main<|\"|>, index:0}]]}}}'\nTransformed: '{\"name\":\"Demo\", connections:{Basic LLM Chain:{\"main\":[[{\"node\":\"X\", type:\"main\", index:0}]]}}}'\nError: Expecting property name enclosed in double quotes: line 1 column 17 (char 16)" }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{name:<|\"|>Demo<|\"|>, connections:{Basic LLM Chain:{main:[[{node:<|\"|>X<|\"|>, type:<|\"|>main<|\"|>, index:0}]]}}}'\nTransformed: '{\"name\":\"Demo\", connections:{Basic LLM Chain:{\"main\":[[{\"node\":\"X\", type:\"main\", index:0}]]}}}'\nError: Expecting property name enclosed in double quotes: line 1 column 17 (char 16)" }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{name:<|\"|>Demo<|\"|>, connections:{Basic LLM Chain:{main:[[{node:<|\"|>X<|\"|>, type:<|\"|>main<|\"|>, index:0}]]}}}'\nTransformed: '{\"name\":\"Demo\", connections:{Basic LLM Chain:{\"main\":[[{\"node\":\"X\", type:\"main\", index:0}]]}}}'\nError: Expecting property name enclosed in double quotes: line 1 column 17 (char 16)" }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{name:<|\"|>Demo<|\"|>, connections:{Basic LLM Chain:{main:[[{node:<|\"|>X<|\"|>, type:<|\"|>main<|\"|>, index:0}]]}}}'\nTransformed: '{\"name\":\"Demo\", connections:{Basic LLM Chain:{\"main\":[[{\"node\":\"X\", type:\"main\", index:0}]]}}}'\nError: Expecting property name enclosed in double quotes: line 1 column 17 (char 16)" }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{name:<|\"|>Demo<|\"|>, connections:{Basic LLM Chain:{main:[[{node:<|\"|>X<|\"|>, type:<|\"|>main<|\"|>, index:0}]]}}}'\nTransformed: '{\"name\":\"Demo\", connections:{Basic LLM Chain:{\"main\":[[{\"node\":\"X\", type:\"main\", index:0}]]}}}'\nError: Expecting property name enclosed in double quotes: line 1 column 17 (char 16)" }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{name:<|\"|>Demo<|\"|>, connections:{Basic LLM Chain:{main:[[{node:<|\"|>X<|\"|>, type:<|\"|>main<|\"|>, index:0}]]}}}'\nTransformed: '{\"name\":\"Demo\", connections:{Basic LLM Chain:{\"main\":[[{\"node\":\"X\", type:\"main\", index:0}]]}}}'\nError: Expecting property name enclosed in double quotes: line 1 column 17 (char 16)" } } } }, { "fixture_id": "gemma4/nested-objects-and-arrays", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null } }, { "fixture_id": "gemma4/edge-values-key-with-space", "family": "gemma4", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{flag:false,key with space:1.5e-05,none:null,obj:{a:[1,<|\"|>t<|\"|>],B:1},s:<|\"|>x,y..." }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{flag:false,key with space:1.5e-05,none:null,obj:{a:[1,<|\"|>t<|\"|>],B:1},s:<|\"|>x,y..." }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{flag:false,key with space:1.5e-05,none:null,obj:{a:[1,<|\"|>t<|\"|>],B:1},s:<|\"|>x,y..." }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{flag:false,key with space:1.5e-05,none:null,obj:{a:[1,<|\"|>t<|\"|>],B:1},s:<|\"|>x,y..." }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{flag:false,key with space:1.5e-05,none:null,obj:{a:[1,<|\"|>t<|\"|>],B:1},s:<|\"|>x,y..." }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{flag:false,key with space:1.5e-05,none:null,obj:{a:[1,<|\"|>t<|\"|>],B:1},s:<|\"|>x,y..." }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{flag:false,key with space:1.5e-05,none:null,obj:{a:[1,<|\"|>t<|\"|>],B:1},s:<|\"|>x,y..." }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{flag:false,key with space:1.5e-05,none:null,obj:{a:[1,<|\"|>t<|\"|>],B:1},s:<|\"|>x,y..." }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{flag:false,key with space:1.5e-05,none:null,obj:{a:[1,<|\"|>t<|\"|>],B:1},s:<|\"|>x,y..." }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{flag:false,key with space:1.5e-05,none:null,obj:{a:[1,<|\"|>t<|\"|>],B:1},s:<|\"|>x,y} \"q\"<|\"|>}'\nTransformed: '{\"flag\":false,key with space:1.5e-05,\"none\":null,\"obj\":{\"a\":[1,\"t\"],\"B\":1},\"s\":\"x,y} \\\\\"q\\\\\"\"}'\nError: Expecting property name enclosed in double quotes: line 1 column 15 (char 14)" }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{flag:false,key with space:1.5e-05,none:null,obj:{a:[1,<|\"|>t<|\"|>],B:1},s:<|\"|>x,y} \"q\"<|\"|>}'\nTransformed: '{\"flag\":false,key with space:1.5e-05,\"none\":null,\"obj\":{\"a\":[1,\"t\"],\"B\":1},\"s\":\"x,y} \\\\\"q\\\\\"\"}'\nError: Expecting property name enclosed in double quotes: line 1 column 15 (char 14)" }, "special": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{flag:false,key with space:1.5e-05,none:null,obj:{a:[1,<|\"|>t<|\"|>],B:1},s:<|\"|>x,y} \"q\"<|\"|>}'\nTransformed: '{\"flag\":false,key with space:1.5e-05,\"none\":null,\"obj\":{\"a\":[1,\"t\"],\"B\":1},\"s\":\"x,y} \\\\\"q\\\\\"\"}'\nError: Expecting property name enclosed in double quotes: line 1 column 15 (char 14)" }, "token": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{flag:false,key with space:1.5e-05,none:null,obj:{a:[1,<|\"|>t<|\"|>],B:1},s:<|\"|>x,y} \"q\"<|\"|>}'\nTransformed: '{\"flag\":false,key with space:1.5e-05,\"none\":null,\"obj\":{\"a\":[1,\"t\"],\"B\":1},\"s\":\"x,y} \\\\\"q\\\\\"\"}'\nError: Expecting property name enclosed in double quotes: line 1 column 15 (char 14)" }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{flag:false,key with space:1.5e-05,none:null,obj:{a:[1,<|\"|>t<|\"|>],B:1},s:<|\"|>x,y} \"q\"<|\"|>}'\nTransformed: '{\"flag\":false,key with space:1.5e-05,\"none\":null,\"obj\":{\"a\":[1,\"t\"],\"B\":1},\"s\":\"x,y} \\\\\"q\\\\\"\"}'\nError: Expecting property name enclosed in double quotes: line 1 column 15 (char 14)" }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{flag:false,key with space:1.5e-05,none:null,obj:{a:[1,<|\"|>t<|\"|>],B:1},s:<|\"|>x,y} \"q\"<|\"|>}'\nTransformed: '{\"flag\":false,key with space:1.5e-05,\"none\":null,\"obj\":{\"a\":[1,\"t\"],\"B\":1},\"s\":\"x,y} \\\\\"q\\\\\"\"}'\nError: Expecting property name enclosed in double quotes: line 1 column 15 (char 14)" }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{flag:false,key with space:1.5e-05,none:null,obj:{a:[1,<|\"|>t<|\"|>],B:1},s:<|\"|>x,y} \"q\"<|\"|>}'\nTransformed: '{\"flag\":false,key with space:1.5e-05,\"none\":null,\"obj\":{\"a\":[1,\"t\"],\"B\":1},\"s\":\"x,y} \\\\\"q\\\\\"\"}'\nError: Expecting property name enclosed in double quotes: line 1 column 15 (char 14)" }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{flag:false,key with space:1.5e-05,none:null,obj:{a:[1,<|\"|>t<|\"|>],B:1},s:<|\"|>x,y} \"q\"<|\"|>}'\nTransformed: '{\"flag\":false,key with space:1.5e-05,\"none\":null,\"obj\":{\"a\":[1,\"t\"],\"B\":1},\"s\":\"x,y} \\\\\"q\\\\\"\"}'\nError: Expecting property name enclosed in double quotes: line 1 column 15 (char 14)" }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{flag:false,key with space:1.5e-05,none:null,obj:{a:[1,<|\"|>t<|\"|>],B:1},s:<|\"|>x,y} \"q\"<|\"|>}'\nTransformed: '{\"flag\":false,key with space:1.5e-05,\"none\":null,\"obj\":{\"a\":[1,\"t\"],\"B\":1},\"s\":\"x,y} \\\\\"q\\\\\"\"}'\nError: Expecting property name enclosed in double quotes: line 1 column 15 (char 14)" } } } }, { "fixture_id": "gemma4/dictsort-case-insensitive-order", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null } }, { "fixture_id": "gemma4/marker-in-arguments", "family": "gemma4", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{text:<|\"|>Wrap calls as <|tool_call>call:f{}\\'\\nTransformed: \\'{\"text\":<|\"|>Wrap c..." }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{text:<|\"|>Wrap calls as <|tool_call>call:f{}\\'\\nTransformed: \\'{\"text\":<|\"|>Wrap c..." }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{text:<|\"|>Wrap calls as <|tool_call>call:f{}\\'\\nTransformed: \\'{\"text\":<|\"|>Wrap c..." }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{text:<|\"|>Wrap calls as <|tool_call>call:f{}\\'\\nTransformed: \\'{\"text\":<|\"|>Wrap c..." }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{text:<|\"|>Wrap calls as <|tool_call>call:f{}\\'\\nTransformed: \\'{\"text\":<|\"|>Wrap c..." }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{text:<|\"|>Wrap calls as <|tool_call>call:f{}\\'\\nTransformed: \\'{\"text\":<|\"|>Wrap c..." }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{text:<|\"|>Wrap calls as <|tool_call>call:f{}\\'\\nTransformed: \\'{\"text\":<|\"|>Wrap c..." }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{text:<|\"|>Wrap calls as <|tool_call>call:f{}\\'\\nTransformed: \\'{\"text\":<|\"|>Wrap c..." }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{text:<|\"|>Wrap calls as <|tool_call>call:f{}\\'\\nTransformed: \\'{\"text\":<|\"|>Wrap c..." }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{text:<|\"|>Wrap calls as <|tool_call>call:f{}'\nTransformed: '{\"text\":<|\"|>Wrap calls as <|tool_call>call:f{}'\nError: Expecting value: line 1 column 9 (char 8)" }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{text:<|\"|>Wrap calls as <|tool_call>call:f{}'\nTransformed: '{\"text\":<|\"|>Wrap calls as <|tool_call>call:f{}'\nError: Expecting value: line 1 column 9 (char 8)" }, "special": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{text:<|\"|>Wrap calls as <|tool_call>call:f{}'\nTransformed: '{\"text\":<|\"|>Wrap calls as <|tool_call>call:f{}'\nError: Expecting value: line 1 column 9 (char 8)" }, "token": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{text:<|\"|>Wrap calls as <|tool_call>call:f{}'\nTransformed: '{\"text\":<|\"|>Wrap calls as <|tool_call>call:f{}'\nError: Expecting value: line 1 column 9 (char 8)" }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{text:<|\"|>Wrap calls as <|tool_call>call:f{}'\nTransformed: '{\"text\":<|\"|>Wrap calls as <|tool_call>call:f{}'\nError: Expecting value: line 1 column 9 (char 8)" }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{text:<|\"|>Wrap calls as <|tool_call>call:f{}'\nTransformed: '{\"text\":<|\"|>Wrap calls as <|tool_call>call:f{}'\nError: Expecting value: line 1 column 9 (char 8)" }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{text:<|\"|>Wrap calls as <|tool_call>call:f{}'\nTransformed: '{\"text\":<|\"|>Wrap calls as <|tool_call>call:f{}'\nError: Expecting value: line 1 column 9 (char 8)" }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{text:<|\"|>Wrap calls as <|tool_call>call:f{}'\nTransformed: '{\"text\":<|\"|>Wrap calls as <|tool_call>call:f{}'\nError: Expecting value: line 1 column 9 (char 8)" }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{text:<|\"|>Wrap calls as <|tool_call>call:f{}'\nTransformed: '{\"text\":<|\"|>Wrap calls as <|tool_call>call:f{}'\nError: Expecting value: line 1 column 9 (char 8)" } } } }, { "fixture_id": "gemma4/unicode-emoji", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null } }, { "fixture_id": "gemma4/numeric-arguments", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null } }, { "fixture_id": "gemma4/string-with-newlines-and-backslashes", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null } }, { "fixture_id": "gemma4/json-text-in-string", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null } }, { "fixture_id": "gemma4/array-of-strings", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null } }, { "fixture_id": "gemma4/long-arguments", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null } }, { "fixture_id": "gemma4/array-of-objects-with-braces", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null } }, { "fixture_id": "gemma4/many-strings-then-string-array", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null } }, { "fixture_id": "gemma4/vllm-no-tool-calls", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null } }, { "fixture_id": "gemma4/vllm-single-call", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null } }, { "fixture_id": "gemma4/vllm-multiple-arguments", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null } }, { "fixture_id": "gemma4/vllm-text-before-call", "family": "gemma4", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you.', got 'Let me check the weather for you. '" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you.', got 'Let me check the weather for you. '" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you.', got 'Let me check the weather for you. '" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you.', got 'Let me check the weather for you. '" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you.', got 'Let me check the weather for you. '" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you.', got 'Let me check the weather for you. '" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you.', got 'Let me check the weather for you. '" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): expected 'Let me check the weather for you.', got 'Let me check the weather for you. '" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Let me check the weather for you.', got 'Let me check the weather for you. '" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Let me check the weather for you.', got 'Let me check the weather for you. '" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Let me check the weather for you.', got 'Let me check the weather for you. '" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Let me check the weather for you.', got 'Let me check the weather for you. '" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Let me check the weather for you.', got 'Let me check the weather for you. '" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Let me check the weather for you.', got 'Let me check the weather for you. '" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Let me check the weather for you.', got 'Let me check the weather for you. '" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Let me check the weather for you.', got 'Let me check the weather for you. '" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null }, "observed": { "nonstream": { "content": "Let me check the weather for you.", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Paris\"}" } ], "exception": null }, "streams": { "one": { "content": "Let me check the weather for you. ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Paris\"}" } ], "exception": null }, "special": { "content": "Let me check the weather for you. ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Paris\"}" } ], "exception": null }, "token": { "content": "Let me check the weather for you. ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Paris\"}" } ], "exception": null }, "rand:1:8": { "content": "Let me check the weather for you. ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Paris\"}" } ], "exception": null }, "rand:2:8": { "content": "Let me check the weather for you. ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Paris\"}" } ], "exception": null }, "rand:3:8": { "content": "Let me check the weather for you. ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Paris\"}" } ], "exception": null }, "rand:4:8": { "content": "Let me check the weather for you. ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Paris\"}" } ], "exception": null }, "rand:5:8": { "content": "Let me check the weather for you. ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Paris\"}" } ], "exception": null } } } }, { "fixture_id": "gemma4/vllm-two-calls", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null } }, { "fixture_id": "gemma4/vllm-nested-arguments", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null } }, { "fixture_id": "gemma4/vllm-number-and-boolean", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null } }, { "fixture_id": "gemma4/vllm-hyphenated-function-name", "family": "gemma4", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'-weather{location:<|\"|>London<|\"|>}\\'\\nTransformed: \\'-weather{\"location\":\"London\"}..." }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'-weather{location:<|\"|>London<|\"|>}\\'\\nTransformed: \\'-weather{\"location\":\"London\"}..." }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'-weather{location:<|\"|>London<|\"|>}\\'\\nTransformed: \\'-weather{\"location\":\"London\"}..." }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'-weather{location:<|\"|>London<|\"|>}\\'\\nTransformed: \\'-weather{\"location\":\"London\"}..." }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'-weather{location:<|\"|>London<|\"|>}\\'\\nTransformed: \\'-weather{\"location\":\"London\"}..." }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'-weather{location:<|\"|>London<|\"|>}\\'\\nTransformed: \\'-weather{\"location\":\"London\"}..." }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'-weather{location:<|\"|>London<|\"|>}\\'\\nTransformed: \\'-weather{\"location\":\"London\"}..." }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'-weather{location:<|\"|>London<|\"|>}\\'\\nTransformed: \\'-weather{\"location\":\"London\"}..." }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'-weather{location:<|\"|>London<|\"|>}\\'\\nTransformed: \\'-weather{\"location\":\"London\"}..." }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '-weather{location:<|\"|>London<|\"|>}'\nTransformed: '-weather{\"location\":\"London\"}'\nError: Expecting value: line 1 column 1 (char 0)" }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '-weather{location:<|\"|>London<|\"|>}'\nTransformed: '-weather{\"location\":\"London\"}'\nError: Expecting value: line 1 column 1 (char 0)" }, "special": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '-weather{location:<|\"|>London<|\"|>}'\nTransformed: '-weather{\"location\":\"London\"}'\nError: Expecting value: line 1 column 1 (char 0)" }, "token": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '-weather{location:<|\"|>London<|\"|>}'\nTransformed: '-weather{\"location\":\"London\"}'\nError: Expecting value: line 1 column 1 (char 0)" }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '-weather{location:<|\"|>London<|\"|>}'\nTransformed: '-weather{\"location\":\"London\"}'\nError: Expecting value: line 1 column 1 (char 0)" }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '-weather{location:<|\"|>London<|\"|>}'\nTransformed: '-weather{\"location\":\"London\"}'\nError: Expecting value: line 1 column 1 (char 0)" }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '-weather{location:<|\"|>London<|\"|>}'\nTransformed: '-weather{\"location\":\"London\"}'\nError: Expecting value: line 1 column 1 (char 0)" }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '-weather{location:<|\"|>London<|\"|>}'\nTransformed: '-weather{\"location\":\"London\"}'\nError: Expecting value: line 1 column 1 (char 0)" }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '-weather{location:<|\"|>London<|\"|>}'\nTransformed: '-weather{\"location\":\"London\"}'\nError: Expecting value: line 1 column 1 (char 0)" } } } }, { "fixture_id": "gemma4/vllm-dotted-function-name", "family": "gemma4", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'.get{location:<|\"|>London<|\"|>}\\'\\nTransformed: \\'.get{\"location\":\"London\"}\\'\\nErro..." }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'.get{location:<|\"|>London<|\"|>}\\'\\nTransformed: \\'.get{\"location\":\"London\"}\\'\\nErro..." }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'.get{location:<|\"|>London<|\"|>}\\'\\nTransformed: \\'.get{\"location\":\"London\"}\\'\\nErro..." }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'.get{location:<|\"|>London<|\"|>}\\'\\nTransformed: \\'.get{\"location\":\"London\"}\\'\\nErro..." }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'.get{location:<|\"|>London<|\"|>}\\'\\nTransformed: \\'.get{\"location\":\"London\"}\\'\\nErro..." }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'.get{location:<|\"|>London<|\"|>}\\'\\nTransformed: \\'.get{\"location\":\"London\"}\\'\\nErro..." }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'.get{location:<|\"|>London<|\"|>}\\'\\nTransformed: \\'.get{\"location\":\"London\"}\\'\\nErro..." }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'.get{location:<|\"|>London<|\"|>}\\'\\nTransformed: \\'.get{\"location\":\"London\"}\\'\\nErro..." }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'.get{location:<|\"|>London<|\"|>}\\'\\nTransformed: \\'.get{\"location\":\"London\"}\\'\\nErro..." }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '.get{location:<|\"|>London<|\"|>}'\nTransformed: '.get{\"location\":\"London\"}'\nError: Expecting value: line 1 column 1 (char 0)" }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '.get{location:<|\"|>London<|\"|>}'\nTransformed: '.get{\"location\":\"London\"}'\nError: Expecting value: line 1 column 1 (char 0)" }, "special": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '.get{location:<|\"|>London<|\"|>}'\nTransformed: '.get{\"location\":\"London\"}'\nError: Expecting value: line 1 column 1 (char 0)" }, "token": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '.get{location:<|\"|>London<|\"|>}'\nTransformed: '.get{\"location\":\"London\"}'\nError: Expecting value: line 1 column 1 (char 0)" }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '.get{location:<|\"|>London<|\"|>}'\nTransformed: '.get{\"location\":\"London\"}'\nError: Expecting value: line 1 column 1 (char 0)" }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '.get{location:<|\"|>London<|\"|>}'\nTransformed: '.get{\"location\":\"London\"}'\nError: Expecting value: line 1 column 1 (char 0)" }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '.get{location:<|\"|>London<|\"|>}'\nTransformed: '.get{\"location\":\"London\"}'\nError: Expecting value: line 1 column 1 (char 0)" }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '.get{location:<|\"|>London<|\"|>}'\nTransformed: '.get{\"location\":\"London\"}'\nError: Expecting value: line 1 column 1 (char 0)" }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '.get{location:<|\"|>London<|\"|>}'\nTransformed: '.get{\"location\":\"London\"}'\nError: Expecting value: line 1 column 1 (char 0)" } } } }, { "fixture_id": "gemma4/vllm-no-arguments", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null } }, { "fixture_id": "gemma4/vllm-reasoning-thought-prefix", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": true }, "stop_token_appended": null } }, { "fixture_id": "gemma4/vllm-reasoning-thought-prefix-multiline", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": true }, "stop_token_appended": null } }, { "fixture_id": "gemma4/sglang-text-before-call", "family": "gemma4", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "soft_pass", "detail": "content (whitespace only): expected 'Some text before ', got 'Some text before'" }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Some text before', got 'Some text before '" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Some text before', got 'Some text before '" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Some text before', got 'Some text before '" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Some text before', got 'Some text before '" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Some text before', got 'Some text before '" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Some text before', got 'Some text before '" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Some text before', got 'Some text before '" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "soft_pass", "detail": "content (whitespace only): nonstream 'Some text before', got 'Some text before '" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null }, "observed": { "nonstream": { "content": "Some text before", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Tokyo\"}" } ], "exception": null }, "streams": { "one": { "content": "Some text before ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Tokyo\"}" } ], "exception": null }, "special": { "content": "Some text before ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Tokyo\"}" } ], "exception": null }, "token": { "content": "Some text before ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Tokyo\"}" } ], "exception": null }, "rand:1:8": { "content": "Some text before ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Tokyo\"}" } ], "exception": null }, "rand:2:8": { "content": "Some text before ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Tokyo\"}" } ], "exception": null }, "rand:3:8": { "content": "Some text before ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Tokyo\"}" } ], "exception": null }, "rand:4:8": { "content": "Some text before ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Tokyo\"}" } ], "exception": null }, "rand:5:8": { "content": "Some text before ", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Tokyo\"}" } ], "exception": null } } } }, { "fixture_id": "gemma4/sglang-text-around-call", "family": "gemma4", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "content: expected 'Some text before after', got 'after'" }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "content: nonstream 'after', got 'Some text before after'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "content: nonstream 'after', got 'Some text before after'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "content: nonstream 'after', got 'Some text before after'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "content: nonstream 'after', got 'Some text before after'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "content: nonstream 'after', got 'Some text before after'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "content: nonstream 'after', got 'Some text before after'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "content: nonstream 'after', got 'Some text before after'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "content: nonstream 'after', got 'Some text before after'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null }, "observed": { "nonstream": { "content": "after", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Tokyo\"}" } ], "exception": null }, "streams": { "one": { "content": "Some text before after", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Tokyo\"}" } ], "exception": null }, "special": { "content": "Some text before after", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Tokyo\"}" } ], "exception": null }, "token": { "content": "Some text before after", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Tokyo\"}" } ], "exception": null }, "rand:1:8": { "content": "Some text before after", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Tokyo\"}" } ], "exception": null }, "rand:2:8": { "content": "Some text before after", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Tokyo\"}" } ], "exception": null }, "rand:3:8": { "content": "Some text before after", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Tokyo\"}" } ], "exception": null }, "rand:4:8": { "content": "Some text before after", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Tokyo\"}" } ], "exception": null }, "rand:5:8": { "content": "Some text before after", "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"location\": \"Tokyo\"}" } ], "exception": null } } } }, { "fixture_id": "gemma4/sglang-nested-array-with-spaces", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null } }, { "fixture_id": "gemma4/sglang-nested-object", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null } }, { "fixture_id": "gemma4/vllm-malformed-no-brace", "family": "gemma4", "status": "fail", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "fail", "detail": "outcome exception \"ValueError: json parser could not parse region as JSON.\\nContent: ' no brace'\\nError: Expecting value: line 1 column 2 (char 1)\"; accepted: ['content_passthrough', 'no_tool_calls']" }, { "check": "expected_error", "strategy": "one", "status": "fail", "detail": "outcome exception \"ValueError: json parser could not parse region as JSON.\\nContent: ' no brace'\\nError: Expecting value: line 1 column 2 (char 1)\"; accepted: ['content_passthrough', 'no_tool_calls']" }, { "check": "expected_error", "strategy": "special", "status": "fail", "detail": "outcome exception \"ValueError: json parser could not parse region as JSON.\\nContent: ' no brace'\\nError: Expecting value: line 1 column 2 (char 1)\"; accepted: ['content_passthrough', 'no_tool_calls']" }, { "check": "expected_error", "strategy": "token", "status": "fail", "detail": "outcome exception \"ValueError: json parser could not parse region as JSON.\\nContent: ' no brace'\\nError: Expecting value: line 1 column 2 (char 1)\"; accepted: ['content_passthrough', 'no_tool_calls']" }, { "check": "expected_error", "strategy": "rand:1:8", "status": "fail", "detail": "outcome exception \"ValueError: json parser could not parse region as JSON.\\nContent: ' no brace'\\nError: Expecting value: line 1 column 2 (char 1)\"; accepted: ['content_passthrough', 'no_tool_calls']" }, { "check": "expected_error", "strategy": "rand:2:8", "status": "fail", "detail": "outcome exception \"ValueError: json parser could not parse region as JSON.\\nContent: ' no brace'\\nError: Expecting value: line 1 column 2 (char 1)\"; accepted: ['content_passthrough', 'no_tool_calls']" }, { "check": "expected_error", "strategy": "rand:3:8", "status": "fail", "detail": "outcome exception \"ValueError: json parser could not parse region as JSON.\\nContent: ' no brace'\\nError: Expecting value: line 1 column 2 (char 1)\"; accepted: ['content_passthrough', 'no_tool_calls']" }, { "check": "expected_error", "strategy": "rand:4:8", "status": "fail", "detail": "outcome exception \"ValueError: json parser could not parse region as JSON.\\nContent: ' no brace'\\nError: Expecting value: line 1 column 2 (char 1)\"; accepted: ['content_passthrough', 'no_tool_calls']" }, { "check": "expected_error", "strategy": "rand:5:8", "status": "fail", "detail": "outcome exception \"ValueError: json parser could not parse region as JSON.\\nContent: ' no brace'\\nError: Expecting value: line 1 column 2 (char 1)\"; accepted: ['content_passthrough', 'no_tool_calls']" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json parser could not parse region as JSON.\nContent: ' no brace'\nError: Expecting value: line 1 column 2 (char 1)" }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json parser could not parse region as JSON.\nContent: ' no brace'\nError: Expecting value: line 1 column 2 (char 1)" }, "special": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json parser could not parse region as JSON.\nContent: ' no brace'\nError: Expecting value: line 1 column 2 (char 1)" }, "token": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json parser could not parse region as JSON.\nContent: ' no brace'\nError: Expecting value: line 1 column 2 (char 1)" }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json parser could not parse region as JSON.\nContent: ' no brace'\nError: Expecting value: line 1 column 2 (char 1)" }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json parser could not parse region as JSON.\nContent: ' no brace'\nError: Expecting value: line 1 column 2 (char 1)" }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json parser could not parse region as JSON.\nContent: ' no brace'\nError: Expecting value: line 1 column 2 (char 1)" }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json parser could not parse region as JSON.\nContent: ' no brace'\nError: Expecting value: line 1 column 2 (char 1)" }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json parser could not parse region as JSON.\nContent: ' no brace'\nError: Expecting value: line 1 column 2 (char 1)" } } } }, { "fixture_id": "gemma4/multi-turn-final-answer", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null } }, { "fixture_id": "gemma4/multi-turn-second-call", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null } }, { "fixture_id": "gemma4/parallel-two-calls-thinking", "family": "gemma4", "status": "soft_pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "expected_match", "strategy": "special", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "expected_match", "strategy": "token", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "expected_match", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "expected_match", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "expected_match", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "expected_match", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "expected_match", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): expected 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "soft_pass", "detail": "reasoning_content (whitespace only): nonstream 'I should call the tools.', got 'I should call the tools.\\n'" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": true }, "stop_token_appended": null }, "observed": { "nonstream": { "content": null, "reasoning_content": "I should call the tools.", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Z\\u00fcrich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"filters\": {\"max\": 3, \"tags\": [\"a\", \"b\"]}, \"query\": \"caf\\u00e9 \\\"best\\\"\"}" } ], "exception": null }, "streams": { "one": { "content": null, "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Z\\u00fcrich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"filters\": {\"max\": 3, \"tags\": [\"a\", \"b\"]}, \"query\": \"caf\\u00e9 \\\"best\\\"\"}" } ], "exception": null }, "special": { "content": null, "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Z\\u00fcrich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"filters\": {\"max\": 3, \"tags\": [\"a\", \"b\"]}, \"query\": \"caf\\u00e9 \\\"best\\\"\"}" } ], "exception": null }, "token": { "content": null, "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Z\\u00fcrich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"filters\": {\"max\": 3, \"tags\": [\"a\", \"b\"]}, \"query\": \"caf\\u00e9 \\\"best\\\"\"}" } ], "exception": null }, "rand:1:8": { "content": null, "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Z\\u00fcrich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"filters\": {\"max\": 3, \"tags\": [\"a\", \"b\"]}, \"query\": \"caf\\u00e9 \\\"best\\\"\"}" } ], "exception": null }, "rand:2:8": { "content": null, "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Z\\u00fcrich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"filters\": {\"max\": 3, \"tags\": [\"a\", \"b\"]}, \"query\": \"caf\\u00e9 \\\"best\\\"\"}" } ], "exception": null }, "rand:3:8": { "content": null, "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Z\\u00fcrich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"filters\": {\"max\": 3, \"tags\": [\"a\", \"b\"]}, \"query\": \"caf\\u00e9 \\\"best\\\"\"}" } ], "exception": null }, "rand:4:8": { "content": null, "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Z\\u00fcrich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"filters\": {\"max\": 3, \"tags\": [\"a\", \"b\"]}, \"query\": \"caf\\u00e9 \\\"best\\\"\"}" } ], "exception": null }, "rand:5:8": { "content": null, "reasoning_content": "I should call the tools.\n", "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Z\\u00fcrich\", \"unit\": \"c\"}" }, { "name": "search", "arguments_raw": "{\"filters\": {\"max\": 3, \"tags\": [\"a\", \"b\"]}, \"query\": \"caf\\u00e9 \\\"best\\\"\"}" } ], "exception": null } } } }, { "fixture_id": "gemma4/parallel-three-calls", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null } }, { "fixture_id": "gemma4/parallel-mixed-with-empty", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "one", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "special", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "token", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "parallel_order", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null } }, { "fixture_id": "gemma4/truncated-mid-string", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null } }, { "fixture_id": "gemma4/truncated-in-name", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null } }, { "fixture_id": "gemma4/truncated-before-close", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_error", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_error", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null } }, { "fixture_id": "gemma4/truncated-in-reasoning", "family": "gemma4", "status": "pass", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "one", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "special", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "token", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "expected_match", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": true }, "stop_token_appended": null } }, { "fixture_id": "gemma4/truncated-second-parallel-call", "family": "gemma4", "status": "fail", "checks": [ { "check": "expected_match", "strategy": "nonstream", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{filters:{max:3,tags:[<|\"|>a<|\"|>,<|\"|>b<|\"|>]},query:\\'\\nTransformed: \\'{\"filters\"..." }, { "check": "expected_match", "strategy": "one", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{filters:{max:3,tags:[<|\"|>a<|\"|>,<|\"|>b<|\"|>]},query:\\'\\nTransformed: \\'{\"filters\"..." }, { "check": "expected_match", "strategy": "special", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{filters:{max:3,tags:[<|\"|>a<|\"|>,<|\"|>b<|\"|>]},query:\\'\\nTransformed: \\'{\"filters\"..." }, { "check": "expected_match", "strategy": "token", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{filters:{max:3,tags:[<|\"|>a<|\"|>,<|\"|>b<|\"|>]},query:\\'\\nTransformed: \\'{\"filters\"..." }, { "check": "expected_match", "strategy": "rand:1:8", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{filters:{max:3,tags:[<|\"|>a<|\"|>,<|\"|>b<|\"|>]},query:\\'\\nTransformed: \\'{\"filters\"..." }, { "check": "expected_match", "strategy": "rand:2:8", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{filters:{max:3,tags:[<|\"|>a<|\"|>,<|\"|>b<|\"|>]},query:\\'\\nTransformed: \\'{\"filters\"..." }, { "check": "expected_match", "strategy": "rand:3:8", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{filters:{max:3,tags:[<|\"|>a<|\"|>,<|\"|>b<|\"|>]},query:\\'\\nTransformed: \\'{\"filters\"..." }, { "check": "expected_match", "strategy": "rand:4:8", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{filters:{max:3,tags:[<|\"|>a<|\"|>,<|\"|>b<|\"|>]},query:\\'\\nTransformed: \\'{\"filters\"..." }, { "check": "expected_match", "strategy": "rand:5:8", "status": "fail", "detail": "parser raised 'ValueError: json: could not parse after dialect transforms.\\nOriginal: \\'{filters:{max:3,tags:[<|\"|>a<|\"|>,<|\"|>b<|\"|>]},query:\\'\\nTransformed: \\'{\"filters\"..." }, { "check": "stream_equals_nonstream", "strategy": "one", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "special", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "token", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:1:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:2:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:3:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:4:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "stream_equals_nonstream", "strategy": "rand:5:8", "status": "fail", "detail": "tool_calls: nonstream [], got ['get_weather']" }, { "check": "no_leakage", "strategy": "nonstream", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "one", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "special", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "token", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "no_leakage", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_json", "strategy": "rand:5:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "one", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "special", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "token", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:1:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:2:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:3:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:4:8", "status": "pass", "detail": null }, { "check": "arguments_schema", "strategy": "rand:5:8", "status": "pass", "detail": null } ], "parser_config": { "tool_parser": "response_template", "reasoning_parser": "response_template", "response_template_source": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475:tokenizer_config.json", "response_template_sha256": "6c8545de7fd056896d82142383de94d98339d44945fe00285f0c4c4ba1ed3a44", "chat_template_sha256": "ae53464bf3be25802b3a5b37def7fd89667067d7577049b3b2d74c4d8de4c6d4", "tokenizer": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475", "tokenizer_mode": "hf", "nonstream_entrypoint": "tokenizer.parse_response(output_ids, prefix=prompt, tools=tools)", "stream_entrypoint": "ResponseParser(response_template, prefix=prompt, tools=tools).feed/.finalize", "stream_detokenizer": "tokenizers.decoders.DecodeStream(skip_special_tokens=False), one id per step", "stream_events": "transformers.cli.serving.utils.response_events_to_chunks", "tools_passed_to_parser": true, "prompt": { "user_message": "Please help with the request, using the available tools if needed.", "add_generation_prompt": true, "enable_thinking": false }, "stop_token_appended": null }, "observed": { "nonstream": { "content": null, "reasoning_content": null, "tool_calls": [], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{filters:{max:3,tags:[<|\"|>a<|\"|>,<|\"|>b<|\"|>]},query:'\nTransformed: '{\"filters\":{\"max\":3,\"tags\":[\"a\",\"b\"]},\"query\":'\nError: Expecting value: line 1 column 47 (char 46)" }, "streams": { "one": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Z\\u00fcrich\", \"unit\": \"c\"}" } ], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{filters:{max:3,tags:[<|\"|>a<|\"|>,<|\"|>b<|\"|>]},query:'\nTransformed: '{\"filters\":{\"max\":3,\"tags\":[\"a\",\"b\"]},\"query\":'\nError: Expecting value: line 1 column 47 (char 46)" }, "special": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Z\\u00fcrich\", \"unit\": \"c\"}" } ], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{filters:{max:3,tags:[<|\"|>a<|\"|>,<|\"|>b<|\"|>]},query:'\nTransformed: '{\"filters\":{\"max\":3,\"tags\":[\"a\",\"b\"]},\"query\":'\nError: Expecting value: line 1 column 47 (char 46)" }, "token": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Z\\u00fcrich\", \"unit\": \"c\"}" } ], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{filters:{max:3,tags:[<|\"|>a<|\"|>,<|\"|>b<|\"|>]},query:'\nTransformed: '{\"filters\":{\"max\":3,\"tags\":[\"a\",\"b\"]},\"query\":'\nError: Expecting value: line 1 column 47 (char 46)" }, "rand:1:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Z\\u00fcrich\", \"unit\": \"c\"}" } ], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{filters:{max:3,tags:[<|\"|>a<|\"|>,<|\"|>b<|\"|>]},query:'\nTransformed: '{\"filters\":{\"max\":3,\"tags\":[\"a\",\"b\"]},\"query\":'\nError: Expecting value: line 1 column 47 (char 46)" }, "rand:2:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Z\\u00fcrich\", \"unit\": \"c\"}" } ], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{filters:{max:3,tags:[<|\"|>a<|\"|>,<|\"|>b<|\"|>]},query:'\nTransformed: '{\"filters\":{\"max\":3,\"tags\":[\"a\",\"b\"]},\"query\":'\nError: Expecting value: line 1 column 47 (char 46)" }, "rand:3:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Z\\u00fcrich\", \"unit\": \"c\"}" } ], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{filters:{max:3,tags:[<|\"|>a<|\"|>,<|\"|>b<|\"|>]},query:'\nTransformed: '{\"filters\":{\"max\":3,\"tags\":[\"a\",\"b\"]},\"query\":'\nError: Expecting value: line 1 column 47 (char 46)" }, "rand:4:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Z\\u00fcrich\", \"unit\": \"c\"}" } ], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{filters:{max:3,tags:[<|\"|>a<|\"|>,<|\"|>b<|\"|>]},query:'\nTransformed: '{\"filters\":{\"max\":3,\"tags\":[\"a\",\"b\"]},\"query\":'\nError: Expecting value: line 1 column 47 (char 46)" }, "rand:5:8": { "content": null, "reasoning_content": null, "tool_calls": [ { "name": "get_weather", "arguments_raw": "{\"city\": \"Z\\u00fcrich\", \"unit\": \"c\"}" } ], "exception": "ValueError: json: could not parse after dialect transforms.\nOriginal: '{filters:{max:3,tags:[<|\"|>a<|\"|>,<|\"|>b<|\"|>]},query:'\nTransformed: '{\"filters\":{\"max\":3,\"tags\":[\"a\",\"b\"]},\"query\":'\nError: Expecting value: line 1 column 47 (char 46)" } } } }, { "fixture_id": "glm/glm45-single-call", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm45-parallel-same-tool", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm45-parallel-mixed-nested", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm45-content-before-call", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm45-no-call", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm45-nested-object", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm45-unicode-emoji", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm45-empty-arguments", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm45-marker-in-arguments", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm45-schema-coercion", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm45-thinking-disabled", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm45-call-order-vs-tool-order", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm47-single-call", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm47-parallel-same-tool", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm47-content-before-call", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm47-no-call", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm47-nested-object", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm47-unicode-emoji", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm47-empty-arguments", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm47-marker-in-arguments", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm47-typed-values", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm47-number-looking-string", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm47-schema-coercion", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm47-string-escapes", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm47-significant-whitespace", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm47-long-arguments", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm47-json-looking-string", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm47-thinking-disabled", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm47-call-order-vs-tool-order", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm47-multi-turn-second-call", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm53-single-call", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm53-parallel-same-tool", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm53-no-call", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm53-schema-coercion", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm45-vllm-newline-format-content-before", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm45-vllm-newline-format-two-calls", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm45-vllm-coerces-schema-types", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm45-vllm-zero-arg-newline", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm45-vllm-with-think-tags", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm45-ollama-content-after-call", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm47-vllm-whitespace-preserved-in-value", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm47-vllm-content-before-no-space", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm47-vllm-whitespace-only-content", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm47-vllm-args-with-newlines", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm47-vllm-reasoning-without-open-tag", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm47-bug-three-calls-same-tool", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm47-truncated-inside-arg-value", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm47-truncated-before-close-call", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm45-truncated-after-arg-key", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm47-truncated-in-reasoning", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "glm/glm47-missing-last-close-arg-value", "family": "glm", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'glm' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/vllm-channel-first-recipient", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/vllm-channel-first-empty-arguments", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/vllm-split-deltas-constrain-nospace", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/vllm-call-on-analysis-channel", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/vllm-preamble-call-end-terminated", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/vllm-sequential-calls", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/vllm-call-then-final", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/vllm-truncated-final", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/vllm-interrupted-final-only", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/vllm-malformed-headers", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/llamacpp-commentary-content-only", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/llamacpp-recipient-in-channel-analysis", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/llamacpp-recipient-in-channel-commentary", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/llamacpp-recipient-in-role-constrain", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/llamacpp-edit-code-arguments", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/llamacpp-builtin-python-recipient", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/llamacpp-stray-commentary-header", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/harmony-test-channel-first-constrain-adjacent", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/harmony-test-role-recipient-constrain-adjacent", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/harmony-spec-receiving-tool-calls", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/harmony-spec-preamble-action-plan", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/bug-unknown-channel-comment", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/bug-garbled-channel-question-marks", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/bug-garbled-channel-commentary-question", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/harmony-single-call", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/harmony-single-call-no-reasoning", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/harmony-compact-json", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/harmony-unicode-emoji", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/harmony-nested-json", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/harmony-empty-arguments", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/harmony-marker-in-arguments", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/harmony-marker-in-reasoning", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/harmony-preamble-then-call", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/harmony-preamble-no-reasoning", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/harmony-no-call-final", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/harmony-final-only", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/harmony-final-json-lookalike", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/harmony-string-escapes", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/harmony-numeric-arguments", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/harmony-long-arguments", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/harmony-multiline-reasoning", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/harmony-content-type-json-text", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/harmony-no-content-type", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/harmony-truncated-in-arguments", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/harmony-truncated-in-reasoning", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/harmony-truncated-in-header", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/hf-template-single-call", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/hf-template-unicode-nested", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/hf-template-empty-arguments", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/hf-template-no-reasoning", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "gpt-oss/hf-template-final-answer", "family": "gpt-oss", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'gpt-oss' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k2i-vllm-single-call-content-before", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k2i-vllm-three-tool-calls", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k2i-vllm-multiline-json", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k2i-vllm-id-without-functions-prefix", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k2i-vllm-noise-between-markers", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k2i-vllm-empty-tool-section", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k2i-vllm-content-after-tool-section", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k3-vllm-no-call-thinking-disabled", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k3-vllm-call-thinking-disabled", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k3-vllm-truncated-tools", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k3-vllm-reasoning-prefix-consumed", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k3-bug-truncated-reasoning-recorded", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k3-bug-response-only-completion", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k26-single-call", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k26-parallel-same-tool", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k26-parallel-three-tools", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k26-content-before-call", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k26-no-call", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k26-nested-json", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k26-unicode-emoji", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k26-empty-arguments", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k26-string-escapes", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k26-marker-in-arguments", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k26-marker-terminator-in-string", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k26-empty-think", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k26-thinking-disabled-call", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k26-multi-turn-second-call", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k2i-single-call", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k2i-content-before-call", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k2i-no-call", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k3-single-call", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k3-parallel-same-tool", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k3-content-before-call", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k3-no-call", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k3-nested-json", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k3-typed-values", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k3-empty-arguments", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k3-string-escapes", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k3-control-marker-text-in-value", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k3-xtml-attr-escaping", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k3-long-arguments", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k3-empty-think", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k3-thinking-disabled-call", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k3-thinking-disabled-no-call", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k3-multi-turn-second-call", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k26-truncated-inside-arguments", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k3-truncated-inside-argument", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k26-truncated-in-reasoning", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "kimi/k3-truncated-in-reasoning", "family": "kimi", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'kimi' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l3-vllm-parameters-key", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l3-vllm-arguments-key", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l3-vllm-deeply-nested", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l3-vllm-brackets-in-strings", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l3-vllm-escaped-quotes", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l3-vllm-json-without-name-is-content", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l3-sglang-python-tag-call", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l3-sglang-text-before-python-tag", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l4-vllm-pythonic-no-call", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l4-vllm-pythonic-more-types", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l4-vllm-pythonic-parameterless", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l4-vllm-pythonic-escaped-strings", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l4-vllm-pythonic-parallel-no-spaces", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l4-vllm-pythonic-python-start-end", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l3-bug-leading-json-object-in-content", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l3-bug-empty-json-object-in-content", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l3-bug-whole-call-single-delta", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l4-bug-leading-underscore-identifier", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l3-single-call", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l3-nested-json", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l3-unicode-emoji", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l3-empty-arguments", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l3-marker-in-arguments", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l3-string-escapes", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l3-typed-values", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l3-long-arguments", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l3-no-call", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l3-json-answer-not-a-call", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l3-multi-turn-second-call", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l3-meta-python-tag-type-function", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l4-meta-pythonic-parallel", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l4-meta-pythonic-int-and-string", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l4-truncated-missing-close-bracket", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "llama/l3-truncated-inside-parameters", "family": "llama", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'llama' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/vllm-v3-no-space", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/vllm-v3-arguments-before-name", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/vllm-v3-name-inside-arguments", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/vllm-v3-parallel", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/vllm-v3-content-before-call", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/vllm-v3-malformed-not-json", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/vllm-v11-single-call", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/vllm-v11-parallel", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/vllm-v11-content-before-call", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/vllm-v13-think-no-end", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/vllm-v11-think-text", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/llamacpp-ministral3-reasoning-call", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/llamacpp-ministral3-parallel", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/llamacpp-ministral3-marker-in-reasoning", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/llamacpp-v11-call-id", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/llamacpp-v11-call-id-parallel", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/llamacpp-devstral-content-before-call", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/bug-devstral-glob-braces-in-arguments", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/v13think-single-call", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/v13think-content-before-call", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/v13think-parallel-calls", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/v13think-no-call", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/v13think-empty-arguments", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/v13think-marker-in-arguments", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/v13think-multiline-reasoning", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/v13think-truncated-in-reasoning", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/v13think-truncated-in-arguments", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/v13think-stop-at-open-marker", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/v13-single-call", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/v13-parallel-calls", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/v13-content-before-call", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/v13-no-call", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/v13-nested-json", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/v13-unicode-emoji", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/v13-string-escapes", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/v13-numeric-arguments", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/v13-long-arguments", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/v13-param-named-name", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/v13-array-lookalike-argument", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/v11-single-call", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/v11-parallel-calls", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/v11-content-before-call", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/v11-no-call", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/v3-single-call", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/v3-parallel-calls", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/v3-empty-arguments", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "mistral/v3-no-call", "family": "mistral", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'mistral' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/single-call-thinking", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/single-call-no-thinking", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/no-call-thinking", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/no-call-no-thinking", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/text-before-call-thinking", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/text-before-call-no-thinking", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/empty-arguments-no-thinking", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/empty-arguments-thinking", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/bug-sglang-30480-truncated-mid-arguments", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/bug-sglang-30480-truncated-at-opener", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/nested-json-deep", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/unicode-emoji-thinking", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/marker-in-arguments", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/string-escapes-code", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/json-string-argument", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/numeric-arguments", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/long-arguments", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/vllm-plain-text-forwarded", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/vllm-bug-19056-boolean-argument", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/vllm-no-newlines-no-spaces", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/vllm-text-then-call-no-separator", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/vllm-two-calls-no-separator", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/vllm-content-and-call-single-chunk", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/sglang-text-before-call-with-space", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/sglang-qwen25-four-parallel-calls", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/vllm-reasoning-multiline", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/vllm-reasoning-without-start-token", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/instruct2507-single-call", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/instruct2507-parallel-calls", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/instruct2507-no-call", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/instruct2507-empty-arguments", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/instruct2507-text-before-call", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/vllm-invalid-json-missing-brace", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/sglang-malformed-json-in-tags", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/multi-turn-second-call-thinking", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/multi-turn-final-answer-thinking", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/parallel-two-calls-thinking", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/parallel-three-calls-no-thinking", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/parallel-mixed-tools-text-before-no-thinking", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/thinking2507-single-call", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/thinking2507-parallel-calls", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/thinking2507-no-call", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/thinking2507-text-before-call", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/thinking2507-multi-turn", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/truncated-mid-arguments-no-thinking", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/truncated-mid-arguments-thinking", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/truncated-after-open-tag", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/truncated-second-parallel-call", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/truncated-in-reasoning", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-hermes/thinking2507-truncated-in-reasoning", "family": "qwen3-hermes", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-hermes' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/llamacpp-q35-reasoning-then-call", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/llamacpp-q35-parallel-no-think", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/llamacpp-q35-python-multiline", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/llamacpp-q35-value-trailing-newline", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/llamacpp-q35-code-leading-indent", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/llamacpp-q35-call-inside-think", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/llamacpp-q35-empty-args", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/llamacpp-q35-reasoning-content-call", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/llamacpp-coder-bare-function", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/llamacpp-coder-bare-function-parallel", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/llamacpp-coder-unicode-cjk", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/llamacpp-coder-html-value", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/llamacpp-coder-array-of-objects", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/vllm-coder-text-before-call-no-newline", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/vllm-coder-anyof-coercion", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/vllm-coder-missing-close-parameter", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/bug-missing-close-parameter-before-function", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/bug-coder-missing-tool-call-opener", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/bug-coder-text-after-call", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/bug-coder-number-outside-int64", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/q38-single-call", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/q38-parallel-calls", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/q38-three-parallel-calls", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/q38-unicode-emoji", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/q38-nested-json", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/q38-empty-arguments", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/q38-marker-in-arguments", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/q38-tool-call-token-in-arguments", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/q38-text-before-call", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/q38-no-call", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/q38-multiline-string", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/q38-whitespace-significant", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/q38-numeric-and-literal-arguments", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/q38-numeric-looking-strings", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/q38-string-escapes", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/q38-long-arguments", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/q38-empty-reasoning", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/q38-thinking-disabled", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/q38-thinking-disabled-parallel", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/q38-thinking-disabled-no-call", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/q38-truncated-in-reasoning", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/q38-truncated-in-parameter", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/q38-truncated-at-tool-call-open", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/q35-python-literals", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/q35-single-call", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/coder-single-call", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/coder-parallel-calls", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/coder-text-before-call", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/coder-no-call", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/coder-python-literals", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/coder-empty-arguments", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/coder-marker-in-arguments", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/coder-next-json-literals", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" }, { "fixture_id": "qwen3-xml/coder-next-parallel-calls", "family": "qwen3-xml", "status": "unsupported", "checks": [], "reason": "no Hub repo of family 'qwen3-xml' ships a response_template in tokenizer_config.json (transformers only parses with a repo-shipped template)" } ], "summary": { "totals": { "pass": 33, "soft_pass": 7, "fail": 8, "error": 0, "unsupported": 421 }, "by_family": { "deepseek": { "pass": 0, "soft_pass": 0, "fail": 0, "error": 0, "unsupported": 85 }, "gemma4": { "pass": 33, "soft_pass": 7, "fail": 8, "error": 0, "unsupported": 0 }, "glm": { "pass": 0, "soft_pass": 0, "fail": 0, "error": 0, "unsupported": 51 }, "gpt-oss": { "pass": 0, "soft_pass": 0, "fail": 0, "error": 0, "unsupported": 51 }, "kimi": { "pass": 0, "soft_pass": 0, "fail": 0, "error": 0, "unsupported": 49 }, "llama": { "pass": 0, "soft_pass": 0, "fail": 0, "error": 0, "unsupported": 34 }, "mistral": { "pass": 0, "soft_pass": 0, "fail": 0, "error": 0, "unsupported": 47 }, "qwen3-hermes": { "pass": 0, "soft_pass": 0, "fail": 0, "error": 0, "unsupported": 50 }, "qwen3-xml": { "pass": 0, "soft_pass": 0, "fail": 0, "error": 0, "unsupported": 54 } } } }