[ { "suite": "publish", "model": "codexresponses.gpt-5.4-mini", "model_slug": "codexresponses-gpt-5-4-mini", "source_kind": "clean-final", "label": "skill-with-shell-codexresponses-gpt-5-4-mini-publication-final", "eval": "numeric-data", "artifact_path": "results/publish/models/codexresponses-gpt-5-4-mini/artifacts/numeric-data.html", "screenshot_desktop_path": "results/publish/models/codexresponses-gpt-5-4-mini/reports/screenshots/numeric-data-desktop.png", "screenshot_mobile_path": "results/publish/models/codexresponses-gpt-5-4-mini/reports/screenshots/numeric-data-mobile.png", "screenshot_deep_path": "results/publish/models/codexresponses-gpt-5-4-mini/reports/screenshots/numeric-data-deep.png", "screenshot_mobile_deep_path": "results/publish/models/codexresponses-gpt-5-4-mini/reports/screenshots/numeric-data-mobile-deep.png", "artifact_bytes": 41655, "generation_ok": true, "generation_duration_s": 233.57, "input_tokens": 257043, "output_tokens": 19565, "total_tokens": 276608, "billing_tokens": 276608, "reasoning_tokens": 13843, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 236032, "total_cache_tokens": 236032, "effective_input_tokens": 21011, "display_input_tokens": 257043, "usage_event_count": 12, "tool_calls": 16, "turn_count": 12, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 2, "self_check_failed_runs": 1, "self_check_successful_runs": 1, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": true, "assistant_turns_trace": 12, "self_check_mode": "run-checker-cli", "self_check_evidence": "ran checker CLI: python /home/shaun/source/birch-html/skill/scripts/check_birch_renderings.py --artifact /home/shaun/source/birch-html/eval-runs/skill-with-shell-codexresponses-gpt-5-4-mini-publica", "deterministic_failures": 0, "deterministic_warnings": 2, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 1, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 1, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 1, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 99, "task_score": 19.8, "task_score_max": 20, "quality_score": 99, "quality_cap_reason": "", "quality_class": "warn" }, { "suite": "publish", "model": "codexresponses.gpt-5.4-mini", "model_slug": "codexresponses-gpt-5-4-mini", "source_kind": "clean-final", "label": "skill-with-shell-codexresponses-gpt-5-4-mini-publication-final", "eval": "code-review", "artifact_path": "results/publish/models/codexresponses-gpt-5-4-mini/artifacts/code-review.html", "screenshot_desktop_path": "results/publish/models/codexresponses-gpt-5-4-mini/reports/screenshots/code-review-desktop.png", "screenshot_mobile_path": "results/publish/models/codexresponses-gpt-5-4-mini/reports/screenshots/code-review-mobile.png", "screenshot_deep_path": "results/publish/models/codexresponses-gpt-5-4-mini/reports/screenshots/code-review-deep.png", "screenshot_mobile_deep_path": "results/publish/models/codexresponses-gpt-5-4-mini/reports/screenshots/code-review-mobile-deep.png", "artifact_bytes": 40247, "generation_ok": true, "generation_duration_s": 251.091, "input_tokens": 1602209, "output_tokens": 16541, "total_tokens": 1618750, "billing_tokens": 1618750, "reasoning_tokens": 10735, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 1516544, "total_cache_tokens": 1516544, "effective_input_tokens": 85665, "display_input_tokens": 1602209, "usage_event_count": 24, "tool_calls": 39, "turn_count": 24, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 3, "self_check_failed_runs": 1, "self_check_successful_runs": 2, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": true, "assistant_turns_trace": 24, "self_check_mode": "checker-cli-error,run-checker-cli", "self_check_evidence": "ran checker CLI: cd /home/shaun/source/birch-html && uv run python skill/scripts/check_birch_renderings.py --help | sed -n '1,220p' | checker CLI usage error | ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-codexresponses-gpt-5-4-mini-publicatio | ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact /home/shaun/source/birch-html/eval-runs/skill-with-shell-codexres", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "codexresponses.gpt-5.4-mini", "model_slug": "codexresponses-gpt-5-4-mini", "source_kind": "clean-final", "label": "skill-with-shell-codexresponses-gpt-5-4-mini-publication-final", "eval": "module-explainer", "artifact_path": "results/publish/models/codexresponses-gpt-5-4-mini/artifacts/module-explainer.html", "screenshot_desktop_path": "results/publish/models/codexresponses-gpt-5-4-mini/reports/screenshots/module-explainer-desktop.png", "screenshot_mobile_path": "results/publish/models/codexresponses-gpt-5-4-mini/reports/screenshots/module-explainer-mobile.png", "screenshot_deep_path": "results/publish/models/codexresponses-gpt-5-4-mini/reports/screenshots/module-explainer-deep.png", "screenshot_mobile_deep_path": "results/publish/models/codexresponses-gpt-5-4-mini/reports/screenshots/module-explainer-mobile-deep.png", "artifact_bytes": 51503, "generation_ok": true, "generation_duration_s": 228.357, "input_tokens": 538144, "output_tokens": 20613, "total_tokens": 558757, "billing_tokens": 558757, "reasoning_tokens": 12973, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 489472, "total_cache_tokens": 489472, "effective_input_tokens": 48672, "display_input_tokens": 538144, "usage_event_count": 14, "tool_calls": 29, "turn_count": 14, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 2, "self_check_failed_runs": 0, "self_check_successful_runs": 2, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 14, "self_check_mode": "checker-shell-reference,read-checker,run-checker-cli", "self_check_evidence": "read /home/shaun/source/birch-html/scripts/check_birch_renderings.py | shell referenced checker: rg -n \"^def (contract_findings|compare_stats|screenshot_findings|artifact_screenshot_findings|geometry_findings|render_markdown|capture|find_chrome|capture_height_for_viewport|css_ | ran checker CLI: mkdir -p /home/shaun/source/birch-html/eval-runs/skill-with-shell-codexresponses-gpt-5-4-mini-publication-final && cat > /home/shaun/source/birch-html/eval-runs/skill-with-shell-co | ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-codexresponses-gpt-5-4-mini-publication-fina", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "codexresponses.gpt-5.4-mini", "model_slug": "codexresponses-gpt-5-4-mini", "source_kind": "clean-final", "label": "skill-with-shell-codexresponses-gpt-5-4-mini-publication-final", "eval": "implementation-plan", "artifact_path": "results/publish/models/codexresponses-gpt-5-4-mini/artifacts/implementation-plan.html", "screenshot_desktop_path": "results/publish/models/codexresponses-gpt-5-4-mini/reports/screenshots/implementation-plan-desktop.png", "screenshot_mobile_path": "results/publish/models/codexresponses-gpt-5-4-mini/reports/screenshots/implementation-plan-mobile.png", "screenshot_deep_path": "results/publish/models/codexresponses-gpt-5-4-mini/reports/screenshots/implementation-plan-deep.png", "screenshot_mobile_deep_path": "results/publish/models/codexresponses-gpt-5-4-mini/reports/screenshots/implementation-plan-mobile-deep.png", "artifact_bytes": 48838, "generation_ok": true, "generation_duration_s": 249.193, "input_tokens": 122451, "output_tokens": 13529, "total_tokens": 135980, "billing_tokens": 135980, "reasoning_tokens": 8129, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 103936, "total_cache_tokens": 103936, "effective_input_tokens": 18515, "display_input_tokens": 122451, "usage_event_count": 8, "tool_calls": 11, "turn_count": 8, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 2, "self_check_failed_runs": 1, "self_check_successful_runs": 1, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": true, "assistant_turns_trace": 8, "self_check_mode": "run-checker-cli", "self_check_evidence": "ran checker CLI: cat > /home/shaun/source/birch-html/eval-runs/skill-with-shell-codexresponses-gpt-5-4-mini-publication-final/implementation-plan.html <<'EOF'\n\n\n /home/shaun/source/birch-html/eval-runs/skill-with-shell-codexresponses-gpt-5-5-publication-final/module-explainer.html <<'EOF'\n\n\n\n /home/shaun/source/birch-html/eval-runs/skill-with-shell-codexre | ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-codexresponses-gpt-5-5-publication-fin | ran checker CLI: cd /home/shaun/source/birch-html && python3 - <<'PY'\nfrom pathlib import Path\np=Path('eval-runs/skill-with-shell-codexresponses-gpt-5-5-publication-final/implementation-plan.html')", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "codexresponses.gpt-5.5", "model_slug": "codexresponses-gpt-5-5", "source_kind": "clean-final", "label": "skill-with-shell-codexresponses-gpt-5-5-publication-final", "eval": "benchmark-comparison", "artifact_path": "results/publish/models/codexresponses-gpt-5-5/artifacts/benchmark-comparison.html", "screenshot_desktop_path": "results/publish/models/codexresponses-gpt-5-5/reports/screenshots/benchmark-comparison-desktop.png", "screenshot_mobile_path": "results/publish/models/codexresponses-gpt-5-5/reports/screenshots/benchmark-comparison-mobile.png", "screenshot_deep_path": "results/publish/models/codexresponses-gpt-5-5/reports/screenshots/benchmark-comparison-deep.png", "screenshot_mobile_deep_path": "results/publish/models/codexresponses-gpt-5-5/reports/screenshots/benchmark-comparison-mobile-deep.png", "artifact_bytes": 53609, "generation_ok": true, "generation_duration_s": 142.604, "input_tokens": 126650, "output_tokens": 6524, "total_tokens": 133174, "billing_tokens": 133174, "reasoning_tokens": 491, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 101376, "total_cache_tokens": 101376, "effective_input_tokens": 25274, "display_input_tokens": 126650, "usage_event_count": 11, "tool_calls": 13, "turn_count": 11, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 1, "self_check_failed_runs": 0, "self_check_successful_runs": 1, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 11, "self_check_mode": "run-checker-cli", "self_check_evidence": "ran checker CLI: cd /home/shaun/source/birch-html && python3 - <<'PY'\nfrom pathlib import Path\np=Path('eval-runs/skill-with-shell-codexresponses-gpt-5-5-publication-final/benchmark-comparison.html' | ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact /home/shaun/source/birch-html/eval-runs/skill-with-shell-codexres", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "codexspark", "model_slug": "codexspark", "source_kind": "clean-final", "label": "skill-with-shell-codexspark-publication-final", "eval": "numeric-data", "artifact_path": "results/publish/models/codexspark/artifacts/numeric-data.html", "screenshot_desktop_path": "results/publish/models/codexspark/reports/screenshots/numeric-data-desktop.png", "screenshot_mobile_path": "results/publish/models/codexspark/reports/screenshots/numeric-data-mobile.png", "screenshot_deep_path": "results/publish/models/codexspark/reports/screenshots/numeric-data-deep.png", "screenshot_mobile_deep_path": "results/publish/models/codexspark/reports/screenshots/numeric-data-mobile-deep.png", "artifact_bytes": 17281, "generation_ok": true, "generation_duration_s": 82.34, "input_tokens": 825347, "output_tokens": 23923, "total_tokens": 849270, "billing_tokens": 849270, "reasoning_tokens": 13374, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 770688, "total_cache_tokens": 770688, "effective_input_tokens": 54659, "display_input_tokens": 825347, "usage_event_count": 32, "tool_calls": 31, "turn_count": 32, "self_check_attempted": false, "self_check_ran": false, "self_check_succeeded": false, "self_check_runs": 0, "self_check_failed_runs": 0, "self_check_successful_runs": 0, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 32, "self_check_mode": "", "self_check_evidence": "", "deterministic_failures": 6, "deterministic_warnings": 2, "vlm_failures": 1, "vlm_warnings": 0, "deterministic_failure_units": 2, "deterministic_warning_units": 1, "vlm_failure_units": 1, "vlm_warning_units": 0, "desktop_failures": 1, "desktop_warnings": 1, "mobile_failures": 2, "mobile_warnings": 0, "deep_failures": 1, "deep_warnings": 1, "mobile_deep_failures": 2, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 35.0, "task_score": 7.0, "task_score_max": 20, "quality_score": 35.0, "quality_cap_reason": "missing_birch_css", "quality_class": "fail" }, { "suite": "publish", "model": "codexspark", "model_slug": "codexspark", "source_kind": "clean-final", "label": "skill-with-shell-codexspark-publication-final", "eval": "code-review", "artifact_path": "results/publish/models/codexspark/artifacts/code-review.html", "screenshot_desktop_path": "results/publish/models/codexspark/reports/screenshots/code-review-desktop.png", "screenshot_mobile_path": "results/publish/models/codexspark/reports/screenshots/code-review-mobile.png", "screenshot_deep_path": "results/publish/models/codexspark/reports/screenshots/code-review-deep.png", "screenshot_mobile_deep_path": "results/publish/models/codexspark/reports/screenshots/code-review-mobile-deep.png", "artifact_bytes": 9658, "generation_ok": false, "generation_duration_s": 60.395, "input_tokens": 1737615, "output_tokens": 21291, "total_tokens": 1758906, "billing_tokens": 1758906, "reasoning_tokens": 17081, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 1702656, "total_cache_tokens": 1702656, "effective_input_tokens": 86941, "display_input_tokens": 1789597, "usage_event_count": 41, "tool_calls": 32, "turn_count": 26, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 3, "self_check_failed_runs": 0, "self_check_successful_runs": 3, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 41, "self_check_mode": "checker-shell-reference,read-checker", "self_check_evidence": "read /home/shaun/source/birch-html/scripts/check_birch_renderings.py | shell referenced checker: nl -ba /home/shaun/source/birch-html/scripts/check_birch_renderings.py | sed -n '1,260p' | shell referenced checker: nl -ba /home/shaun/source/birch-html/scripts/check_birch_renderings.py | sed -n '260,560p' | shell referenced checker: nl -ba /home/shaun/source/birch-html/scripts/check_birch_renderings.py | sed -n '560,920p' | shell referenced checker: nl -ba /home/shaun/source/birch-html/scripts/check_birch_renderings.py | sed -n '920,1320p'", "deterministic_failures": 8, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 2, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 2, "desktop_warnings": 0, "mobile_failures": 2, "mobile_warnings": 0, "deep_failures": 2, "deep_warnings": 0, "mobile_deep_failures": 2, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 35.0, "task_score": 7.0, "task_score_max": 20, "quality_score": 35.0, "quality_cap_reason": "missing_birch_css", "quality_class": "fail" }, { "suite": "publish", "model": "codexspark", "model_slug": "codexspark", "source_kind": "clean-final", "label": "skill-with-shell-codexspark-publication-final", "eval": "module-explainer", "artifact_path": "results/publish/models/codexspark/artifacts/module-explainer.html", "screenshot_desktop_path": "results/publish/models/codexspark/reports/screenshots/module-explainer-desktop.png", "screenshot_mobile_path": "results/publish/models/codexspark/reports/screenshots/module-explainer-mobile.png", "screenshot_deep_path": "results/publish/models/codexspark/reports/screenshots/module-explainer-deep.png", "screenshot_mobile_deep_path": "results/publish/models/codexspark/reports/screenshots/module-explainer-mobile-deep.png", "artifact_bytes": 16366, "generation_ok": false, "generation_duration_s": 87.747, "input_tokens": 2740590, "output_tokens": 27049, "total_tokens": 2767639, "billing_tokens": 2767639, "reasoning_tokens": 15704, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 2024320, "total_cache_tokens": 2024320, "effective_input_tokens": 202803, "display_input_tokens": 2227123, "usage_event_count": 35, "tool_calls": 51, "turn_count": 42, "self_check_attempted": true, "self_check_ran": false, "self_check_succeeded": false, "self_check_runs": 0, "self_check_failed_runs": 0, "self_check_successful_runs": 0, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 35, "self_check_mode": "checker-shell-reference,read-checker,run-checker-cli", "self_check_evidence": "read /home/shaun/source/birch-html/scripts/check_birch_renderings.py | shell referenced checker: cd /home/shaun/source/birch-html && rg -n \"class\\s*=\\\"(flow-node|flow-edge|flow-list|flow-step|metric-row|chart-panel|finding|code-block|copyable|timeline)\" styles/birch-system.css | shell referenced checker: cd /home/shaun/source/birch-html && wc -l scripts/check_birch_renderings.py | shell referenced checker: cd /home/shaun/source/birch-html && mkdir -p eval-runs/skill-with-shell-codexspark-publication-final && cat > eval-runs/skill-with-shell-codexspark-publication-final/module-explain | ran checker CLI: cd /home/shaun/source/birch-html && cat > eval-runs/skill-with-shell-codexspark-publication-final/module-explainer.html <<'EOF'\n\n\n\n /home/shaun/source/birch-html/eval-runs/skill-with-shell-codexspark-publication-final/implementation-plan.html <<'EOF'\n\n\n\n &1 && echo \"---\" && head -5 eval-runs/skill-with-shell-deepseek-publication-final/benchmark-compari", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "gemini35flash", "model_slug": "gemini35flash", "source_kind": "clean-final", "label": "skill-with-shell-gemini35flash-publication-final", "eval": "numeric-data", "artifact_path": "results/publish/models/gemini35flash/artifacts/numeric-data.html", "screenshot_desktop_path": "results/publish/models/gemini35flash/reports/screenshots/numeric-data-desktop.png", "screenshot_mobile_path": "results/publish/models/gemini35flash/reports/screenshots/numeric-data-mobile.png", "screenshot_deep_path": "results/publish/models/gemini35flash/reports/screenshots/numeric-data-deep.png", "screenshot_mobile_deep_path": "results/publish/models/gemini35flash/reports/screenshots/numeric-data-mobile-deep.png", "artifact_bytes": 53215, "generation_ok": true, "generation_duration_s": 114.216, "input_tokens": 1371616, "output_tokens": 5260, "total_tokens": 1376876, "billing_tokens": 1376876, "reasoning_tokens": 12418, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 1116684, "total_cache_tokens": 1116684, "effective_input_tokens": 254932, "display_input_tokens": 1371616, "usage_event_count": 29, "tool_calls": 28, "turn_count": 29, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 2, "self_check_failed_runs": 1, "self_check_successful_runs": 1, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": true, "assistant_turns_trace": 29, "self_check_mode": "run-checker-cli", "self_check_evidence": "ran checker CLI: uv run --with pillow python3 skill/scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-gemini35flash-publication-final/numeric-data.html | ran checker CLI: uv run --with pillow python3 skill/scripts/check_birch_renderings.py --artifact /home/shaun/source/birch-html/eval-runs/skill-with-shell-gemini35flash-publication-final/numeric-dat", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "gemini35flash", "model_slug": "gemini35flash", "source_kind": "clean-final", "label": "skill-with-shell-gemini35flash-publication-final", "eval": "code-review", "artifact_path": "results/publish/models/gemini35flash/artifacts/code-review.html", "screenshot_desktop_path": "results/publish/models/gemini35flash/reports/screenshots/code-review-desktop.png", "screenshot_mobile_path": "results/publish/models/gemini35flash/reports/screenshots/code-review-mobile.png", "screenshot_deep_path": "results/publish/models/gemini35flash/reports/screenshots/code-review-deep.png", "screenshot_mobile_deep_path": "results/publish/models/gemini35flash/reports/screenshots/code-review-mobile-deep.png", "artifact_bytes": 53047, "generation_ok": true, "generation_duration_s": 193.238, "input_tokens": 1684136, "output_tokens": 6902, "total_tokens": 1691038, "billing_tokens": 1691038, "reasoning_tokens": 23273, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 1424691, "total_cache_tokens": 1424691, "effective_input_tokens": 259445, "display_input_tokens": 1684136, "usage_event_count": 34, "tool_calls": 33, "turn_count": 34, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 3, "self_check_failed_runs": 1, "self_check_successful_runs": 2, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": true, "assistant_turns_trace": 34, "self_check_mode": "checker-cli-error,run-checker-cli", "self_check_evidence": "ran checker CLI: python3 /home/shaun/source/birch-html/skill/scripts/check_birch_renderings.py --help | checker CLI usage error | ran checker CLI: python3 /home/shaun/source/birch-html/skill/scripts/check_birch_renderings.py --artifact /home/shaun/source/birch-html/eval-runs/skill-with-shell-gemini35flash-publication-final/co | ran checker CLI: python3 /home/shaun/source/birch-html/skill/scripts/check_birch_renderings.py --no-capture --artifact /home/shaun/source/birch-html/eval-runs/skill-with-shell-gemini35flash-publica", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "gemini35flash", "model_slug": "gemini35flash", "source_kind": "clean-final", "label": "skill-with-shell-gemini35flash-publication-final", "eval": "module-explainer", "artifact_path": "results/publish/models/gemini35flash/artifacts/module-explainer.html", "screenshot_desktop_path": "results/publish/models/gemini35flash/reports/screenshots/module-explainer-desktop.png", "screenshot_mobile_path": "results/publish/models/gemini35flash/reports/screenshots/module-explainer-mobile.png", "screenshot_deep_path": "results/publish/models/gemini35flash/reports/screenshots/module-explainer-deep.png", "screenshot_mobile_deep_path": "results/publish/models/gemini35flash/reports/screenshots/module-explainer-mobile-deep.png", "artifact_bytes": 57420, "generation_ok": true, "generation_duration_s": 203.178, "input_tokens": 2196880, "output_tokens": 10222, "total_tokens": 2207102, "billing_tokens": 2207102, "reasoning_tokens": 22501, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 1965131, "total_cache_tokens": 1965131, "effective_input_tokens": 231749, "display_input_tokens": 2196880, "usage_event_count": 33, "tool_calls": 32, "turn_count": 33, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": false, "self_check_runs": 2, "self_check_failed_runs": 2, "self_check_successful_runs": 0, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 33, "self_check_mode": "read-checker,run-checker-cli", "self_check_evidence": "read scripts/check_birch_renderings.py | ran checker CLI: python3 scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-gemini35flash-publication-final/module-explainer.html", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "gemini35flash", "model_slug": "gemini35flash", "source_kind": "clean-final", "label": "skill-with-shell-gemini35flash-publication-final", "eval": "implementation-plan", "artifact_path": "results/publish/models/gemini35flash/artifacts/implementation-plan.html", "screenshot_desktop_path": "results/publish/models/gemini35flash/reports/screenshots/implementation-plan-desktop.png", "screenshot_mobile_path": "results/publish/models/gemini35flash/reports/screenshots/implementation-plan-mobile.png", "screenshot_deep_path": "results/publish/models/gemini35flash/reports/screenshots/implementation-plan-deep.png", "screenshot_mobile_deep_path": "results/publish/models/gemini35flash/reports/screenshots/implementation-plan-mobile-deep.png", "artifact_bytes": 49628, "generation_ok": true, "generation_duration_s": 201.715, "input_tokens": 2346900, "output_tokens": 9173, "total_tokens": 2356073, "billing_tokens": 2356073, "reasoning_tokens": 15150, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 2043078, "total_cache_tokens": 2043078, "effective_input_tokens": 303822, "display_input_tokens": 2346900, "usage_event_count": 34, "tool_calls": 33, "turn_count": 34, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 5, "self_check_failed_runs": 4, "self_check_successful_runs": 1, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 34, "self_check_mode": "checker-cli-error,run-checker-cli", "self_check_evidence": "ran checker CLI: python3 skill/scripts/check_birch_renderings.py --help | checker CLI usage error | ran checker CLI: python3 skill/scripts/check_birch_renderings.py --artifact temp_plan.html | ran checker CLI: python3 skill/scripts/check_birch_renderings.py --artifact /home/shaun/source/birch-html/temp_plan.html | ran checker CLI: python3 skill/scripts/check_birch_renderings.py --artifact /home/shaun/source/birch-html/eval-runs/skill-with-shell-gemini35flash-publication-final/implementation-plan.html", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "gemini35flash", "model_slug": "gemini35flash", "source_kind": "clean-final", "label": "skill-with-shell-gemini35flash-publication-final", "eval": "benchmark-comparison", "artifact_path": "results/publish/models/gemini35flash/artifacts/benchmark-comparison.html", "screenshot_desktop_path": "results/publish/models/gemini35flash/reports/screenshots/benchmark-comparison-desktop.png", "screenshot_mobile_path": "results/publish/models/gemini35flash/reports/screenshots/benchmark-comparison-mobile.png", "screenshot_deep_path": "results/publish/models/gemini35flash/reports/screenshots/benchmark-comparison-deep.png", "screenshot_mobile_deep_path": "results/publish/models/gemini35flash/reports/screenshots/benchmark-comparison-mobile-deep.png", "artifact_bytes": 97390, "generation_ok": true, "generation_duration_s": 62.077, "input_tokens": 495825, "output_tokens": 829, "total_tokens": 496654, "billing_tokens": 496654, "reasoning_tokens": 4961, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 387138, "total_cache_tokens": 387138, "effective_input_tokens": 108687, "display_input_tokens": 495825, "usage_event_count": 17, "tool_calls": 16, "turn_count": 17, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": false, "self_check_runs": 1, "self_check_failed_runs": 1, "self_check_successful_runs": 0, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 17, "self_check_mode": "run-checker-cli", "self_check_evidence": "ran checker CLI: python3 /home/shaun/source/birch-html/skill/scripts/check_birch_renderings.py --artifact /home/shaun/source/birch-html/eval-runs/skill-with-shell-gemini35flash-publication-final/be", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "glm51", "model_slug": "glm51", "source_kind": "clean-final", "label": "skill-with-shell-glm51-publication-final", "eval": "numeric-data", "artifact_path": "results/publish/models/glm51/artifacts/numeric-data.html", "screenshot_desktop_path": "results/publish/models/glm51/reports/screenshots/numeric-data-desktop.png", "screenshot_mobile_path": "results/publish/models/glm51/reports/screenshots/numeric-data-mobile.png", "screenshot_deep_path": "results/publish/models/glm51/reports/screenshots/numeric-data-deep.png", "screenshot_mobile_deep_path": "results/publish/models/glm51/reports/screenshots/numeric-data-mobile-deep.png", "artifact_bytes": 62971, "generation_ok": true, "generation_duration_s": 300.114, "input_tokens": 459899, "output_tokens": 16275, "total_tokens": 476174, "billing_tokens": 476174, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 369152, "total_cache_tokens": 369152, "effective_input_tokens": 90747, "display_input_tokens": 459899, "usage_event_count": 15, "tool_calls": 16, "turn_count": 15, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": false, "self_check_runs": 1, "self_check_failed_runs": 1, "self_check_successful_runs": 0, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 15, "self_check_mode": "run-checker-cli", "self_check_evidence": "ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-glm51-publication-final/numeric-data.h", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 2, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 1, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 99, "task_score": 19.8, "task_score_max": 20, "quality_score": 99, "quality_cap_reason": "", "quality_class": "warn" }, { "suite": "publish", "model": "glm51", "model_slug": "glm51", "source_kind": "clean-final", "label": "skill-with-shell-glm51-publication-final", "eval": "code-review", "artifact_path": "results/publish/models/glm51/artifacts/code-review.html", "screenshot_desktop_path": "results/publish/models/glm51/reports/screenshots/code-review-desktop.png", "screenshot_mobile_path": "results/publish/models/glm51/reports/screenshots/code-review-mobile.png", "screenshot_deep_path": "results/publish/models/glm51/reports/screenshots/code-review-deep.png", "screenshot_mobile_deep_path": "results/publish/models/glm51/reports/screenshots/code-review-mobile-deep.png", "artifact_bytes": 48933, "generation_ok": true, "generation_duration_s": 133.324, "input_tokens": 254816, "output_tokens": 8008, "total_tokens": 262824, "billing_tokens": 262824, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 202560, "total_cache_tokens": 202560, "effective_input_tokens": 52256, "display_input_tokens": 254816, "usage_event_count": 11, "tool_calls": 13, "turn_count": 11, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 1, "self_check_failed_runs": 0, "self_check_successful_runs": 1, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 11, "self_check_mode": "run-checker-cli", "self_check_evidence": "ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-glm51-publication-final/code-review.ht", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 2, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 1, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 92, "task_score": 18.4, "task_score_max": 20, "quality_score": 92, "quality_cap_reason": "", "quality_class": "fail" }, { "suite": "publish", "model": "glm51", "model_slug": "glm51", "source_kind": "clean-final", "label": "skill-with-shell-glm51-publication-final", "eval": "module-explainer", "artifact_path": "results/publish/models/glm51/artifacts/module-explainer.html", "screenshot_desktop_path": "results/publish/models/glm51/reports/screenshots/module-explainer-desktop.png", "screenshot_mobile_path": "results/publish/models/glm51/reports/screenshots/module-explainer-mobile.png", "screenshot_deep_path": "results/publish/models/glm51/reports/screenshots/module-explainer-deep.png", "screenshot_mobile_deep_path": "results/publish/models/glm51/reports/screenshots/module-explainer-mobile-deep.png", "artifact_bytes": 54229, "generation_ok": true, "generation_duration_s": 94.822, "input_tokens": 358438, "output_tokens": 6652, "total_tokens": 365090, "billing_tokens": 365090, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 254656, "total_cache_tokens": 254656, "effective_input_tokens": 103782, "display_input_tokens": 358438, "usage_event_count": 9, "tool_calls": 15, "turn_count": 9, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 1, "self_check_failed_runs": 0, "self_check_successful_runs": 1, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 9, "self_check_mode": "read-checker,run-checker-cli", "self_check_evidence": "read /home/shaun/source/birch-html/scripts/check_birch_renderings.py | ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-glm51-publication-final/module-explainer.htm", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "glm51", "model_slug": "glm51", "source_kind": "clean-final", "label": "skill-with-shell-glm51-publication-final", "eval": "implementation-plan", "artifact_path": "results/publish/models/glm51/artifacts/implementation-plan.html", "screenshot_desktop_path": "results/publish/models/glm51/reports/screenshots/implementation-plan-desktop.png", "screenshot_mobile_path": "results/publish/models/glm51/reports/screenshots/implementation-plan-mobile.png", "screenshot_deep_path": "results/publish/models/glm51/reports/screenshots/implementation-plan-deep.png", "screenshot_mobile_deep_path": "results/publish/models/glm51/reports/screenshots/implementation-plan-mobile-deep.png", "artifact_bytes": 60535, "generation_ok": true, "generation_duration_s": 90.03, "input_tokens": 210191, "output_tokens": 7574, "total_tokens": 217765, "billing_tokens": 217765, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 180736, "total_cache_tokens": 180736, "effective_input_tokens": 29455, "display_input_tokens": 210191, "usage_event_count": 15, "tool_calls": 16, "turn_count": 15, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 2, "self_check_failed_runs": 0, "self_check_successful_runs": 2, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 15, "self_check_mode": "run-checker-cli", "self_check_evidence": "ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-glm51-publication-final/implementation", "deterministic_failures": 2, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 2, "deterministic_failure_units": 1, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 1, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 1, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 1, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 93, "task_score": 18.6, "task_score_max": 20, "quality_score": 93, "quality_cap_reason": "", "quality_class": "fail" }, { "suite": "publish", "model": "glm51", "model_slug": "glm51", "source_kind": "clean-final", "label": "skill-with-shell-glm51-publication-final", "eval": "benchmark-comparison", "artifact_path": "results/publish/models/glm51/artifacts/benchmark-comparison.html", "screenshot_desktop_path": "results/publish/models/glm51/reports/screenshots/benchmark-comparison-desktop.png", "screenshot_mobile_path": "results/publish/models/glm51/reports/screenshots/benchmark-comparison-mobile.png", "screenshot_deep_path": "results/publish/models/glm51/reports/screenshots/benchmark-comparison-deep.png", "screenshot_mobile_deep_path": "results/publish/models/glm51/reports/screenshots/benchmark-comparison-mobile-deep.png", "artifact_bytes": 64863, "generation_ok": true, "generation_duration_s": 149.159, "input_tokens": 274201, "output_tokens": 14416, "total_tokens": 288617, "billing_tokens": 288617, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 214336, "total_cache_tokens": 214336, "effective_input_tokens": 59865, "display_input_tokens": 274201, "usage_event_count": 12, "tool_calls": 14, "turn_count": 12, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 1, "self_check_failed_runs": 0, "self_check_successful_runs": 1, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 12, "self_check_mode": "run-checker-cli", "self_check_evidence": "ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-glm51-publication-final/benchmark-comp", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "glm52", "model_slug": "glm52", "source_kind": "clean-final", "label": "skill-with-shell-glm52-publication-final", "eval": "numeric-data", "artifact_path": "results/publish/models/glm52/artifacts/numeric-data.html", "screenshot_desktop_path": "results/publish/models/glm52/reports/screenshots/numeric-data-desktop.png", "screenshot_mobile_path": "results/publish/models/glm52/reports/screenshots/numeric-data-mobile.png", "screenshot_deep_path": "results/publish/models/glm52/reports/screenshots/numeric-data-deep.png", "screenshot_mobile_deep_path": "results/publish/models/glm52/reports/screenshots/numeric-data-mobile-deep.png", "artifact_bytes": 51395, "generation_ok": true, "generation_duration_s": 274.73, "input_tokens": 271862, "output_tokens": 16133, "total_tokens": 287995, "billing_tokens": 287995, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 239936, "total_cache_tokens": 239936, "effective_input_tokens": 31926, "display_input_tokens": 271862, "usage_event_count": 14, "tool_calls": 17, "turn_count": 14, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 1, "self_check_failed_runs": 0, "self_check_successful_runs": 1, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 14, "self_check_mode": "run-checker-cli", "self_check_evidence": "ran checker CLI: cd /home/shaun/source/birch-html && uv run skill/scripts/finish_birch_html.py eval-runs/skill-with-shell-glm52-publication-final/numeric-data.html && echo \"=== finished ===\" && uv | ran checker CLI: cd /home/shaun/source/birch-html && F=eval-runs/skill-with-shell-glm52-publication-final/numeric-data.html\necho \"=== my local style block (after system block) ===\"\npython3 - \"$F\" <", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "glm52", "model_slug": "glm52", "source_kind": "clean-final", "label": "skill-with-shell-glm52-publication-final", "eval": "code-review", "artifact_path": "results/publish/models/glm52/artifacts/code-review.html", "screenshot_desktop_path": "results/publish/models/glm52/reports/screenshots/code-review-desktop.png", "screenshot_mobile_path": "results/publish/models/glm52/reports/screenshots/code-review-mobile.png", "screenshot_deep_path": "results/publish/models/glm52/reports/screenshots/code-review-deep.png", "screenshot_mobile_deep_path": "results/publish/models/glm52/reports/screenshots/code-review-mobile-deep.png", "artifact_bytes": 50845, "generation_ok": true, "generation_duration_s": 771.097, "input_tokens": 1249523, "output_tokens": 43260, "total_tokens": 1292783, "billing_tokens": 1292783, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 1177792, "total_cache_tokens": 1177792, "effective_input_tokens": 71731, "display_input_tokens": 1249523, "usage_event_count": 25, "tool_calls": 32, "turn_count": 25, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 3, "self_check_failed_runs": 0, "self_check_successful_runs": 3, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 25, "self_check_mode": "checker-cli-error,run-checker-cli", "self_check_evidence": "ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-glm52-publication-final/code-review.ht | ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact \"$(pwd)/eval-runs/skill-with-shell-glm52-publication-final/code-r | ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --help 2>&1 | rg -i 'viewport|artifact|mobile|width' | head; echo \"=== run m | checker CLI usage error", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "glm52", "model_slug": "glm52", "source_kind": "clean-final", "label": "skill-with-shell-glm52-publication-final", "eval": "module-explainer", "artifact_path": "results/publish/models/glm52/artifacts/module-explainer.html", "screenshot_desktop_path": "results/publish/models/glm52/reports/screenshots/module-explainer-desktop.png", "screenshot_mobile_path": "results/publish/models/glm52/reports/screenshots/module-explainer-mobile.png", "screenshot_deep_path": "results/publish/models/glm52/reports/screenshots/module-explainer-deep.png", "screenshot_mobile_deep_path": "results/publish/models/glm52/reports/screenshots/module-explainer-mobile-deep.png", "artifact_bytes": 59952, "generation_ok": true, "generation_duration_s": 751.331, "input_tokens": 1204327, "output_tokens": 40435, "total_tokens": 1244762, "billing_tokens": 1244762, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 1118592, "total_cache_tokens": 1118592, "effective_input_tokens": 85735, "display_input_tokens": 1204327, "usage_event_count": 22, "tool_calls": 32, "turn_count": 22, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 2, "self_check_failed_runs": 0, "self_check_successful_runs": 2, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 22, "self_check_mode": "read-checker,run-checker-cli", "self_check_evidence": "read /home/shaun/source/birch-html/scripts/check_birch_renderings.py | ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-glm52-publication-final/module-explainer.htm", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "glm52", "model_slug": "glm52", "source_kind": "clean-final", "label": "skill-with-shell-glm52-publication-final", "eval": "implementation-plan", "artifact_path": "results/publish/models/glm52/artifacts/implementation-plan.html", "screenshot_desktop_path": "results/publish/models/glm52/reports/screenshots/implementation-plan-desktop.png", "screenshot_mobile_path": "results/publish/models/glm52/reports/screenshots/implementation-plan-mobile.png", "screenshot_deep_path": "results/publish/models/glm52/reports/screenshots/implementation-plan-deep.png", "screenshot_mobile_deep_path": "results/publish/models/glm52/reports/screenshots/implementation-plan-mobile-deep.png", "artifact_bytes": 56320, "generation_ok": true, "generation_duration_s": 456.209, "input_tokens": 991570, "output_tokens": 24123, "total_tokens": 1015693, "billing_tokens": 1015693, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 911168, "total_cache_tokens": 911168, "effective_input_tokens": 80402, "display_input_tokens": 991570, "usage_event_count": 18, "tool_calls": 26, "turn_count": 18, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 3, "self_check_failed_runs": 0, "self_check_successful_runs": 3, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 18, "self_check_mode": "read-checker,run-checker-cli", "self_check_evidence": "read /home/shaun/source/birch-html/skill/scripts/check_birch_renderings.py | ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-glm52-publication-final/implementation | ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact /home/shaun/source/birch-html/eval-runs/skill-with-shell-glm52-pu", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "glm52", "model_slug": "glm52", "source_kind": "clean-final", "label": "skill-with-shell-glm52-publication-final", "eval": "benchmark-comparison", "artifact_path": "results/publish/models/glm52/artifacts/benchmark-comparison.html", "screenshot_desktop_path": "results/publish/models/glm52/reports/screenshots/benchmark-comparison-desktop.png", "screenshot_mobile_path": "results/publish/models/glm52/reports/screenshots/benchmark-comparison-mobile.png", "screenshot_deep_path": "results/publish/models/glm52/reports/screenshots/benchmark-comparison-deep.png", "screenshot_mobile_deep_path": "results/publish/models/glm52/reports/screenshots/benchmark-comparison-mobile-deep.png", "artifact_bytes": 60487, "generation_ok": true, "generation_duration_s": 380.184, "input_tokens": 522022, "output_tokens": 23534, "total_tokens": 545556, "billing_tokens": 545556, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 459648, "total_cache_tokens": 459648, "effective_input_tokens": 62374, "display_input_tokens": 522022, "usage_event_count": 16, "tool_calls": 19, "turn_count": 16, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 2, "self_check_failed_runs": 0, "self_check_successful_runs": 2, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 16, "self_check_mode": "run-checker-cli", "self_check_evidence": "ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-glm52-publication-final/benchmark-comp | ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact \"$(pwd)/eval-runs/skill-with-shell-glm52-publication-final/benchm", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "gpt-5.3-codex", "model_slug": "gpt-5-3-codex", "source_kind": "clean-final", "label": "skill-with-shell-gpt-5-3-codex-publication-final", "eval": "numeric-data", "artifact_path": "results/publish/models/gpt-5-3-codex/artifacts/numeric-data.html", "screenshot_desktop_path": "results/publish/models/gpt-5-3-codex/reports/screenshots/numeric-data-desktop.png", "screenshot_mobile_path": "results/publish/models/gpt-5-3-codex/reports/screenshots/numeric-data-mobile.png", "screenshot_deep_path": "results/publish/models/gpt-5-3-codex/reports/screenshots/numeric-data-deep.png", "screenshot_mobile_deep_path": "results/publish/models/gpt-5-3-codex/reports/screenshots/numeric-data-mobile-deep.png", "artifact_bytes": 40305, "generation_ok": true, "generation_duration_s": 63.372, "input_tokens": 91503, "output_tokens": 5097, "total_tokens": 96600, "billing_tokens": 96600, "reasoning_tokens": 1083, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 76800, "total_cache_tokens": 76800, "effective_input_tokens": 14703, "display_input_tokens": 91503, "usage_event_count": 8, "tool_calls": 11, "turn_count": 8, "self_check_attempted": false, "self_check_ran": false, "self_check_succeeded": false, "self_check_runs": 0, "self_check_failed_runs": 0, "self_check_successful_runs": 0, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 8, "self_check_mode": "", "self_check_evidence": "", "deterministic_failures": 2, "deterministic_warnings": 2, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 1, "deterministic_warning_units": 1, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 1, "mobile_failures": 1, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 1, "mobile_deep_failures": 1, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 93, "task_score": 18.6, "task_score_max": 20, "quality_score": 93, "quality_cap_reason": "", "quality_class": "fail" }, { "suite": "publish", "model": "gpt-5.3-codex", "model_slug": "gpt-5-3-codex", "source_kind": "clean-final", "label": "skill-with-shell-gpt-5-3-codex-publication-final", "eval": "code-review", "artifact_path": "results/publish/models/gpt-5-3-codex/artifacts/code-review.html", "screenshot_desktop_path": "results/publish/models/gpt-5-3-codex/reports/screenshots/code-review-desktop.png", "screenshot_mobile_path": "results/publish/models/gpt-5-3-codex/reports/screenshots/code-review-mobile.png", "screenshot_deep_path": "results/publish/models/gpt-5-3-codex/reports/screenshots/code-review-deep.png", "screenshot_mobile_deep_path": "results/publish/models/gpt-5-3-codex/reports/screenshots/code-review-mobile-deep.png", "artifact_bytes": 39494, "generation_ok": true, "generation_duration_s": 94.334, "input_tokens": 461816, "output_tokens": 6027, "total_tokens": 467843, "billing_tokens": 467843, "reasoning_tokens": 2855, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 384640, "total_cache_tokens": 384640, "effective_input_tokens": 77176, "display_input_tokens": 461816, "usage_event_count": 17, "tool_calls": 18, "turn_count": 17, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": false, "self_check_runs": 1, "self_check_failed_runs": 1, "self_check_successful_runs": 0, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 17, "self_check_mode": "run-checker-cli", "self_check_evidence": "ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-gpt-5-3-codex-publication-final/code-r", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "gpt-5.3-codex", "model_slug": "gpt-5-3-codex", "source_kind": "clean-final", "label": "skill-with-shell-gpt-5-3-codex-publication-final", "eval": "module-explainer", "artifact_path": "results/publish/models/gpt-5-3-codex/artifacts/module-explainer.html", "screenshot_desktop_path": "results/publish/models/gpt-5-3-codex/reports/screenshots/module-explainer-desktop.png", "screenshot_mobile_path": "results/publish/models/gpt-5-3-codex/reports/screenshots/module-explainer-mobile.png", "screenshot_deep_path": "results/publish/models/gpt-5-3-codex/reports/screenshots/module-explainer-deep.png", "screenshot_mobile_deep_path": "results/publish/models/gpt-5-3-codex/reports/screenshots/module-explainer-mobile-deep.png", "artifact_bytes": 46290, "generation_ok": true, "generation_duration_s": 93.641, "input_tokens": 555669, "output_tokens": 7177, "total_tokens": 562846, "billing_tokens": 562846, "reasoning_tokens": 1701, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 450304, "total_cache_tokens": 450304, "effective_input_tokens": 105365, "display_input_tokens": 555669, "usage_event_count": 17, "tool_calls": 23, "turn_count": 17, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 2, "self_check_failed_runs": 1, "self_check_successful_runs": 1, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": true, "assistant_turns_trace": 17, "self_check_mode": "checker-cli-error,checker-shell-reference,read-checker,run-checker-cli", "self_check_evidence": "read /home/shaun/source/birch-html/scripts/check_birch_renderings.py | shell referenced checker: rg '^def ' -n /home/shaun/source/birch-html/scripts/check_birch_renderings.py | ran checker CLI: mkdir -p /home/shaun/source/birch-html/eval-runs/skill-with-shell-gpt-5-3-codex-publication-final && cat > /home/shaun/source/birch-html/eval-runs/skill-with-shell-gpt-5-3-codex-pu | ran checker CLI: uv run --with pillow python /home/shaun/source/birch-html/scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-gpt-5-3-codex-publication-final/module-explainer.h | checker CLI usage error", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 1, "vlm_warnings": 1, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 1, "vlm_warning_units": 1, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 91, "task_score": 18.2, "task_score_max": 20, "quality_score": 91, "quality_cap_reason": "", "quality_class": "fail" }, { "suite": "publish", "model": "gpt-5.3-codex", "model_slug": "gpt-5-3-codex", "source_kind": "clean-final", "label": "skill-with-shell-gpt-5-3-codex-publication-final", "eval": "implementation-plan", "artifact_path": "results/publish/models/gpt-5-3-codex/artifacts/implementation-plan.html", "screenshot_desktop_path": "results/publish/models/gpt-5-3-codex/reports/screenshots/implementation-plan-desktop.png", "screenshot_mobile_path": "results/publish/models/gpt-5-3-codex/reports/screenshots/implementation-plan-mobile.png", "screenshot_deep_path": "results/publish/models/gpt-5-3-codex/reports/screenshots/implementation-plan-deep.png", "screenshot_mobile_deep_path": "results/publish/models/gpt-5-3-codex/reports/screenshots/implementation-plan-mobile-deep.png", "artifact_bytes": 45485, "generation_ok": true, "generation_duration_s": 59.362, "input_tokens": 90659, "output_tokens": 4766, "total_tokens": 95425, "billing_tokens": 95425, "reasoning_tokens": 589, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 71168, "total_cache_tokens": 71168, "effective_input_tokens": 19491, "display_input_tokens": 90659, "usage_event_count": 9, "tool_calls": 10, "turn_count": 9, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 2, "self_check_failed_runs": 1, "self_check_successful_runs": 1, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": true, "assistant_turns_trace": 9, "self_check_mode": "run-checker-cli", "self_check_evidence": "ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-gpt-5-3-codex-publication-final/implem | ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact /home/shaun/source/birch-html/eval-runs/skill-with-shell-gpt-5-3-", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "gpt-5.3-codex", "model_slug": "gpt-5-3-codex", "source_kind": "clean-final", "label": "skill-with-shell-gpt-5-3-codex-publication-final", "eval": "benchmark-comparison", "artifact_path": "results/publish/models/gpt-5-3-codex/artifacts/benchmark-comparison.html", "screenshot_desktop_path": "results/publish/models/gpt-5-3-codex/reports/screenshots/benchmark-comparison-desktop.png", "screenshot_mobile_path": "results/publish/models/gpt-5-3-codex/reports/screenshots/benchmark-comparison-mobile.png", "screenshot_deep_path": "results/publish/models/gpt-5-3-codex/reports/screenshots/benchmark-comparison-deep.png", "screenshot_mobile_deep_path": "results/publish/models/gpt-5-3-codex/reports/screenshots/benchmark-comparison-mobile-deep.png", "artifact_bytes": 46793, "generation_ok": true, "generation_duration_s": 61.812, "input_tokens": 60483, "output_tokens": 5615, "total_tokens": 66098, "billing_tokens": 66098, "reasoning_tokens": 746, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 53376, "total_cache_tokens": 53376, "effective_input_tokens": 7107, "display_input_tokens": 60483, "usage_event_count": 7, "tool_calls": 8, "turn_count": 7, "self_check_attempted": false, "self_check_ran": false, "self_check_succeeded": false, "self_check_runs": 0, "self_check_failed_runs": 0, "self_check_successful_runs": 0, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 7, "self_check_mode": "", "self_check_evidence": "", "deterministic_failures": 4, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 2, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 2, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 2, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 88, "task_score": 17.6, "task_score_max": 20, "quality_score": 88, "quality_cap_reason": "", "quality_class": "fail" }, { "suite": "publish", "model": "grok-4.3", "model_slug": "grok-4-3", "source_kind": "clean-final", "label": "skill-with-shell-grok-4-3-publication-final", "eval": "numeric-data", "artifact_path": "results/publish/models/grok-4-3/artifacts/numeric-data.html", "screenshot_desktop_path": "results/publish/models/grok-4-3/reports/screenshots/numeric-data-desktop.png", "screenshot_mobile_path": "results/publish/models/grok-4-3/reports/screenshots/numeric-data-mobile.png", "screenshot_deep_path": "results/publish/models/grok-4-3/reports/screenshots/numeric-data-deep.png", "screenshot_mobile_deep_path": "results/publish/models/grok-4-3/reports/screenshots/numeric-data-mobile-deep.png", "artifact_bytes": 36903, "generation_ok": true, "generation_duration_s": 49.028, "input_tokens": 73338, "output_tokens": 3307, "total_tokens": 76645, "billing_tokens": 76645, "reasoning_tokens": 925, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 62720, "total_cache_tokens": 62720, "effective_input_tokens": 10618, "display_input_tokens": 73338, "usage_event_count": 10, "tool_calls": 9, "turn_count": 10, "self_check_attempted": false, "self_check_ran": false, "self_check_succeeded": false, "self_check_runs": 0, "self_check_failed_runs": 0, "self_check_successful_runs": 0, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 10, "self_check_mode": "", "self_check_evidence": "", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "grok-4.3", "model_slug": "grok-4-3", "source_kind": "clean-final", "label": "skill-with-shell-grok-4-3-publication-final", "eval": "code-review", "artifact_path": "results/publish/models/grok-4-3/artifacts/code-review.html", "screenshot_desktop_path": "results/publish/models/grok-4-3/reports/screenshots/code-review-desktop.png", "screenshot_mobile_path": "results/publish/models/grok-4-3/reports/screenshots/code-review-mobile.png", "screenshot_deep_path": "results/publish/models/grok-4-3/reports/screenshots/code-review-deep.png", "screenshot_mobile_deep_path": "results/publish/models/grok-4-3/reports/screenshots/code-review-mobile-deep.png", "artifact_bytes": 38297, "generation_ok": true, "generation_duration_s": 55.392, "input_tokens": 190492, "output_tokens": 4553, "total_tokens": 195045, "billing_tokens": 195045, "reasoning_tokens": 2340, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 147520, "total_cache_tokens": 147520, "effective_input_tokens": 42972, "display_input_tokens": 190492, "usage_event_count": 11, "tool_calls": 10, "turn_count": 11, "self_check_attempted": false, "self_check_ran": false, "self_check_succeeded": false, "self_check_runs": 0, "self_check_failed_runs": 0, "self_check_successful_runs": 0, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 11, "self_check_mode": "", "self_check_evidence": "", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "grok-4.3", "model_slug": "grok-4-3", "source_kind": "clean-final", "label": "skill-with-shell-grok-4-3-publication-final", "eval": "module-explainer", "artifact_path": "results/publish/models/grok-4-3/artifacts/module-explainer.html", "screenshot_desktop_path": "results/publish/models/grok-4-3/reports/screenshots/module-explainer-desktop.png", "screenshot_mobile_path": "results/publish/models/grok-4-3/reports/screenshots/module-explainer-mobile.png", "screenshot_deep_path": "results/publish/models/grok-4-3/reports/screenshots/module-explainer-deep.png", "screenshot_mobile_deep_path": "results/publish/models/grok-4-3/reports/screenshots/module-explainer-mobile-deep.png", "artifact_bytes": 9279, "generation_ok": false, "generation_duration_s": 40.052, "input_tokens": 125766, "output_tokens": 3826, "total_tokens": 129592, "billing_tokens": 129592, "reasoning_tokens": 1202, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 46784, "total_cache_tokens": 46784, "effective_input_tokens": 53433, "display_input_tokens": 100217, "usage_event_count": 15, "tool_calls": 6, "turn_count": 7, "self_check_attempted": true, "self_check_ran": false, "self_check_succeeded": false, "self_check_runs": 0, "self_check_failed_runs": 0, "self_check_successful_runs": 0, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 15, "self_check_mode": "read-checker", "self_check_evidence": "read /home/shaun/source/birch-html/scripts/check_birch_renderings.py", "deterministic_failures": 8, "deterministic_warnings": 0, "vlm_failures": 3, "vlm_warnings": 0, "deterministic_failure_units": 2, "deterministic_warning_units": 0, "vlm_failure_units": 2, "vlm_warning_units": 0, "desktop_failures": 2, "desktop_warnings": 0, "mobile_failures": 2, "mobile_warnings": 0, "deep_failures": 2, "deep_warnings": 0, "mobile_deep_failures": 2, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 35.0, "task_score": 7.0, "task_score_max": 20, "quality_score": 35.0, "quality_cap_reason": "missing_birch_css", "quality_class": "fail" }, { "suite": "publish", "model": "grok-4.3", "model_slug": "grok-4-3", "source_kind": "clean-final", "label": "skill-with-shell-grok-4-3-publication-final", "eval": "implementation-plan", "artifact_path": "results/publish/models/grok-4-3/artifacts/implementation-plan.html", "screenshot_desktop_path": "results/publish/models/grok-4-3/reports/screenshots/implementation-plan-desktop.png", "screenshot_mobile_path": "results/publish/models/grok-4-3/reports/screenshots/implementation-plan-mobile.png", "screenshot_deep_path": "results/publish/models/grok-4-3/reports/screenshots/implementation-plan-deep.png", "screenshot_mobile_deep_path": "results/publish/models/grok-4-3/reports/screenshots/implementation-plan-mobile-deep.png", "artifact_bytes": 16152, "generation_ok": false, "generation_duration_s": 41.596, "input_tokens": 32235, "output_tokens": 5236, "total_tokens": 37471, "billing_tokens": 37471, "reasoning_tokens": 1207, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 39488, "total_cache_tokens": 39488, "effective_input_tokens": 20479, "display_input_tokens": 59967, "usage_event_count": 8, "tool_calls": 4, "turn_count": 5, "self_check_attempted": false, "self_check_ran": false, "self_check_succeeded": false, "self_check_runs": 0, "self_check_failed_runs": 0, "self_check_successful_runs": 0, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 8, "self_check_mode": "", "self_check_evidence": "", "deterministic_failures": 4, "deterministic_warnings": 0, "vlm_failures": 4, "vlm_warnings": 0, "deterministic_failure_units": 1, "deterministic_warning_units": 0, "vlm_failure_units": 1, "vlm_warning_units": 0, "desktop_failures": 1, "desktop_warnings": 0, "mobile_failures": 1, "mobile_warnings": 0, "deep_failures": 1, "deep_warnings": 0, "mobile_deep_failures": 1, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 20.0, "task_score": 4.0, "task_score_max": 20, "quality_score": 20.0, "quality_cap_reason": "missing_birch_css_and_visibly_unstyled", "quality_class": "fail" }, { "suite": "publish", "model": "grok-4.3", "model_slug": "grok-4-3", "source_kind": "clean-final", "label": "skill-with-shell-grok-4-3-publication-final", "eval": "benchmark-comparison", "artifact_path": "results/publish/models/grok-4-3/artifacts/benchmark-comparison.html", "screenshot_desktop_path": "results/publish/models/grok-4-3/reports/screenshots/benchmark-comparison-desktop.png", "screenshot_mobile_path": "results/publish/models/grok-4-3/reports/screenshots/benchmark-comparison-mobile.png", "screenshot_deep_path": "results/publish/models/grok-4-3/reports/screenshots/benchmark-comparison-deep.png", "screenshot_mobile_deep_path": "results/publish/models/grok-4-3/reports/screenshots/benchmark-comparison-mobile-deep.png", "artifact_bytes": 10364, "generation_ok": false, "generation_duration_s": 98.19, "input_tokens": 153411, "output_tokens": 7388, "total_tokens": 160799, "billing_tokens": 160799, "reasoning_tokens": 2517, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 39488, "total_cache_tokens": 39488, "effective_input_tokens": 6645, "display_input_tokens": 46133, "usage_event_count": 8, "tool_calls": 15, "turn_count": 16, "self_check_attempted": false, "self_check_ran": false, "self_check_succeeded": false, "self_check_runs": 0, "self_check_failed_runs": 0, "self_check_successful_runs": 0, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 8, "self_check_mode": "", "self_check_evidence": "", "deterministic_failures": 4, "deterministic_warnings": 0, "vlm_failures": 4, "vlm_warnings": 1, "deterministic_failure_units": 1, "deterministic_warning_units": 0, "vlm_failure_units": 1, "vlm_warning_units": 1, "desktop_failures": 1, "desktop_warnings": 0, "mobile_failures": 1, "mobile_warnings": 0, "deep_failures": 1, "deep_warnings": 0, "mobile_deep_failures": 1, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 35.0, "task_score": 7.0, "task_score_max": 20, "quality_score": 35.0, "quality_cap_reason": "missing_birch_css", "quality_class": "fail" }, { "suite": "publish", "model": "haiku45", "model_slug": "haiku45", "source_kind": "clean-final", "label": "skill-with-shell-haiku45-publication-final", "eval": "numeric-data", "artifact_path": "results/publish/models/haiku45/artifacts/numeric-data.html", "screenshot_desktop_path": "results/publish/models/haiku45/reports/screenshots/numeric-data-desktop.png", "screenshot_mobile_path": "results/publish/models/haiku45/reports/screenshots/numeric-data-mobile.png", "screenshot_deep_path": "results/publish/models/haiku45/reports/screenshots/numeric-data-deep.png", "screenshot_mobile_deep_path": "results/publish/models/haiku45/reports/screenshots/numeric-data-mobile-deep.png", "artifact_bytes": 23937, "generation_ok": false, "generation_duration_s": 67.62, "input_tokens": 119520, "output_tokens": 7707, "total_tokens": 127227, "billing_tokens": 127227, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 7297, "cache_write_tokens": 12081, "cache_hit_tokens": 0, "total_cache_tokens": 19378, "effective_input_tokens": 11280, "display_input_tokens": 30658, "usage_event_count": 4, "tool_calls": 9, "turn_count": 10, "self_check_attempted": false, "self_check_ran": false, "self_check_succeeded": false, "self_check_runs": 0, "self_check_failed_runs": 0, "self_check_successful_runs": 0, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 4, "self_check_mode": "run-checker-cli", "self_check_evidence": "ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-haiku45-publication-final/numeric-data", "deterministic_failures": 16, "deterministic_warnings": 12, "vlm_failures": 1, "vlm_warnings": 0, "deterministic_failure_units": 4, "deterministic_warning_units": 3, "vlm_failure_units": 1, "vlm_warning_units": 0, "desktop_failures": 4, "desktop_warnings": 3, "mobile_failures": 4, "mobile_warnings": 3, "deep_failures": 4, "deep_warnings": 3, "mobile_deep_failures": 4, "mobile_deep_warnings": 3, "artifact_present": true, "artifact_score_100": 35.0, "task_score": 7.0, "task_score_max": 20, "quality_score": 35.0, "quality_cap_reason": "missing_birch_css", "quality_class": "fail" }, { "suite": "publish", "model": "haiku45", "model_slug": "haiku45", "source_kind": "clean-final", "label": "skill-with-shell-haiku45-publication-final", "eval": "code-review", "artifact_path": "results/publish/models/haiku45/artifacts/code-review.html", "screenshot_desktop_path": "results/publish/models/haiku45/reports/screenshots/code-review-desktop.png", "screenshot_mobile_path": "results/publish/models/haiku45/reports/screenshots/code-review-mobile.png", "screenshot_deep_path": "results/publish/models/haiku45/reports/screenshots/code-review-deep.png", "screenshot_mobile_deep_path": "results/publish/models/haiku45/reports/screenshots/code-review-mobile-deep.png", "artifact_bytes": 53526, "generation_ok": true, "generation_duration_s": 94.461, "input_tokens": 301467, "output_tokens": 10117, "total_tokens": 311584, "billing_tokens": 311584, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 228528, "cache_write_tokens": 34499, "cache_hit_tokens": 0, "total_cache_tokens": 263027, "effective_input_tokens": 38440, "display_input_tokens": 301467, "usage_event_count": 11, "tool_calls": 11, "turn_count": 11, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 1, "self_check_failed_runs": 0, "self_check_successful_runs": 1, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 11, "self_check_mode": "run-checker-cli", "self_check_evidence": "ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact /home/shaun/source/birch-html/eval-runs/skill-with-shell-haiku45-", "deterministic_failures": 6, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 2, "deterministic_failure_units": 2, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 1, "desktop_failures": 1, "desktop_warnings": 0, "mobile_failures": 2, "mobile_warnings": 0, "deep_failures": 1, "deep_warnings": 0, "mobile_deep_failures": 2, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 87, "task_score": 17.4, "task_score_max": 20, "quality_score": 87, "quality_cap_reason": "", "quality_class": "fail" }, { "suite": "publish", "model": "haiku45", "model_slug": "haiku45", "source_kind": "clean-final", "label": "skill-with-shell-haiku45-publication-final", "eval": "module-explainer", "artifact_path": "results/publish/models/haiku45/artifacts/module-explainer.html", "screenshot_desktop_path": "results/publish/models/haiku45/reports/screenshots/module-explainer-desktop.png", "screenshot_mobile_path": "results/publish/models/haiku45/reports/screenshots/module-explainer-mobile.png", "screenshot_deep_path": "results/publish/models/haiku45/reports/screenshots/module-explainer-deep.png", "screenshot_mobile_deep_path": "results/publish/models/haiku45/reports/screenshots/module-explainer-mobile-deep.png", "artifact_bytes": 57853, "generation_ok": false, "generation_duration_s": 75.42, "input_tokens": 211164, "output_tokens": 9407, "total_tokens": 220571, "billing_tokens": 220571, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 55031, "cache_hit_tokens": 0, "total_cache_tokens": 55031, "effective_input_tokens": 80985, "display_input_tokens": 136016, "usage_event_count": 3, "tool_calls": 10, "turn_count": 6, "self_check_attempted": true, "self_check_ran": false, "self_check_succeeded": false, "self_check_runs": 0, "self_check_failed_runs": 0, "self_check_successful_runs": 0, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 3, "self_check_mode": "read-checker", "self_check_evidence": "read /home/shaun/source/birch-html/scripts/check_birch_renderings.py", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "haiku45", "model_slug": "haiku45", "source_kind": "clean-final", "label": "skill-with-shell-haiku45-publication-final", "eval": "implementation-plan", "artifact_path": "results/publish/models/haiku45/artifacts/implementation-plan.html", "screenshot_desktop_path": "results/publish/models/haiku45/reports/screenshots/implementation-plan-desktop.png", "screenshot_mobile_path": "results/publish/models/haiku45/reports/screenshots/implementation-plan-mobile.png", "screenshot_deep_path": "results/publish/models/haiku45/reports/screenshots/implementation-plan-deep.png", "screenshot_mobile_deep_path": "results/publish/models/haiku45/reports/screenshots/implementation-plan-mobile-deep.png", "artifact_bytes": 50641, "generation_ok": true, "generation_duration_s": 67.418, "input_tokens": 123711, "output_tokens": 7166, "total_tokens": 130877, "billing_tokens": 130877, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 91600, "cache_write_tokens": 16126, "cache_hit_tokens": 0, "total_cache_tokens": 107726, "effective_input_tokens": 15985, "display_input_tokens": 123711, "usage_event_count": 9, "tool_calls": 9, "turn_count": 9, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 1, "self_check_failed_runs": 0, "self_check_successful_runs": 1, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 9, "self_check_mode": "run-checker-cli", "self_check_evidence": "ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact /home/shaun/source/birch-html/eval-runs/skill-with-shell-haiku45-", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "haiku45", "model_slug": "haiku45", "source_kind": "clean-final", "label": "skill-with-shell-haiku45-publication-final", "eval": "benchmark-comparison", "artifact_path": "results/publish/models/haiku45/artifacts/benchmark-comparison.html", "screenshot_desktop_path": "results/publish/models/haiku45/reports/screenshots/benchmark-comparison-desktop.png", "screenshot_mobile_path": "results/publish/models/haiku45/reports/screenshots/benchmark-comparison-mobile.png", "screenshot_deep_path": "results/publish/models/haiku45/reports/screenshots/benchmark-comparison-deep.png", "screenshot_mobile_deep_path": "results/publish/models/haiku45/reports/screenshots/benchmark-comparison-mobile-deep.png", "artifact_bytes": 49137, "generation_ok": true, "generation_duration_s": 65.28, "input_tokens": 151349, "output_tokens": 7796, "total_tokens": 159145, "billing_tokens": 159145, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 122743, "cache_write_tokens": 12640, "cache_hit_tokens": 0, "total_cache_tokens": 135383, "effective_input_tokens": 15966, "display_input_tokens": 151349, "usage_event_count": 11, "tool_calls": 10, "turn_count": 11, "self_check_attempted": false, "self_check_ran": false, "self_check_succeeded": false, "self_check_runs": 0, "self_check_failed_runs": 0, "self_check_successful_runs": 0, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 11, "self_check_mode": "", "self_check_evidence": "", "deterministic_failures": 4, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 3, "deterministic_failure_units": 1, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 1, "desktop_failures": 1, "desktop_warnings": 0, "mobile_failures": 1, "mobile_warnings": 0, "deep_failures": 1, "deep_warnings": 0, "mobile_deep_failures": 1, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 93, "task_score": 18.6, "task_score_max": 20, "quality_score": 93, "quality_cap_reason": "", "quality_class": "fail" }, { "suite": "publish", "model": "kimi", "model_slug": "kimi", "source_kind": "clean-final", "label": "skill-with-shell-kimi-publication-final", "eval": "numeric-data", "artifact_path": "results/publish/models/kimi/artifacts/numeric-data.html", "screenshot_desktop_path": "results/publish/models/kimi/reports/screenshots/numeric-data-desktop.png", "screenshot_mobile_path": "results/publish/models/kimi/reports/screenshots/numeric-data-mobile.png", "screenshot_deep_path": "results/publish/models/kimi/reports/screenshots/numeric-data-deep.png", "screenshot_mobile_deep_path": "results/publish/models/kimi/reports/screenshots/numeric-data-mobile-deep.png", "artifact_bytes": 67620, "generation_ok": true, "generation_duration_s": 194.344, "input_tokens": 470039, "output_tokens": 5317, "total_tokens": 475356, "billing_tokens": 475356, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 425472, "total_cache_tokens": 425472, "effective_input_tokens": 44567, "display_input_tokens": 470039, "usage_event_count": 20, "tool_calls": 23, "turn_count": 20, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 3, "self_check_failed_runs": 1, "self_check_successful_runs": 2, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": true, "assistant_turns_trace": 20, "self_check_mode": "run-checker-cli", "self_check_evidence": "ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-kimi-publication-final/numeric-data.ht | ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact /home/shaun/source/birch-html/eval-runs/skill-with-shell-kimi-pub", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "kimi", "model_slug": "kimi", "source_kind": "clean-final", "label": "skill-with-shell-kimi-publication-final", "eval": "code-review", "artifact_path": "results/publish/models/kimi/artifacts/code-review.html", "screenshot_desktop_path": "results/publish/models/kimi/reports/screenshots/code-review-desktop.png", "screenshot_mobile_path": "results/publish/models/kimi/reports/screenshots/code-review-mobile.png", "screenshot_deep_path": "results/publish/models/kimi/reports/screenshots/code-review-deep.png", "screenshot_mobile_deep_path": "results/publish/models/kimi/reports/screenshots/code-review-mobile-deep.png", "artifact_bytes": 44300, "generation_ok": true, "generation_duration_s": 627.536, "input_tokens": 1248543, "output_tokens": 24596, "total_tokens": 1273139, "billing_tokens": 1273139, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 1192448, "total_cache_tokens": 1192448, "effective_input_tokens": 56095, "display_input_tokens": 1248543, "usage_event_count": 33, "tool_calls": 36, "turn_count": 33, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 2, "self_check_failed_runs": 1, "self_check_successful_runs": 1, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": true, "assistant_turns_trace": 33, "self_check_mode": "checker-shell-reference,read-checker,run-checker-cli", "self_check_evidence": "read /home/shaun/source/birch-html/skill/scripts/check_birch_renderings.py | shell referenced checker: grep -n \"CANDLE_CLASSES\\|BIRCH_CLASSES\\|LAYOUT_CLASSES\\|SEMANTIC_CLASSES\" /home/shaun/source/birch-html/skill/scripts/check_birch_renderings.py | head -20 | shell referenced checker: grep -n \"callout\" /home/shaun/source/birch-html/skill/scripts/check_birch_renderings.py | shell referenced checker: grep -n \"eyebrow\\|lede\\|muted\\|caption\\|subtle\\|note\\|entity\\|label-cell\" /home/shaun/source/birch-html/skill/scripts/check_birch_renderings.py | head -20 | shell referenced checker: grep -n \"code-block\" /home/shaun/source/birch-html/skill/scripts/check_birch_renderings.py | head -20 | shell referenced checker: grep -n \"data-tone\" /home/shaun/source/birch-html/skill/scripts/check_birch_renderings.py | head -20", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "kimi", "model_slug": "kimi", "source_kind": "clean-final", "label": "skill-with-shell-kimi-publication-final", "eval": "module-explainer", "artifact_path": "results/publish/models/kimi/artifacts/module-explainer.html", "screenshot_desktop_path": "results/publish/models/kimi/reports/screenshots/module-explainer-desktop.png", "screenshot_mobile_path": "results/publish/models/kimi/reports/screenshots/module-explainer-mobile.png", "screenshot_deep_path": "results/publish/models/kimi/reports/screenshots/module-explainer-deep.png", "screenshot_mobile_deep_path": "results/publish/models/kimi/reports/screenshots/module-explainer-mobile-deep.png", "artifact_bytes": 17730, "generation_ok": false, "generation_duration_s": 142.653, "input_tokens": 54919, "output_tokens": 5427, "total_tokens": 60346, "billing_tokens": 60346, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 0, "total_cache_tokens": 0, "effective_input_tokens": 54919, "display_input_tokens": 54919, "usage_event_count": 5, "tool_calls": 10, "turn_count": 5, "self_check_attempted": true, "self_check_ran": false, "self_check_succeeded": false, "self_check_runs": 0, "self_check_failed_runs": 0, "self_check_successful_runs": 0, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 5, "self_check_mode": "read-checker", "self_check_evidence": "read /home/shaun/source/birch-html/scripts/check_birch_renderings.py", "deterministic_failures": 6, "deterministic_warnings": 0, "vlm_failures": 7, "vlm_warnings": 1, "deterministic_failure_units": 2, "deterministic_warning_units": 0, "vlm_failure_units": 3, "vlm_warning_units": 1, "desktop_failures": 1, "desktop_warnings": 0, "mobile_failures": 2, "mobile_warnings": 0, "deep_failures": 1, "deep_warnings": 0, "mobile_deep_failures": 2, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 20.0, "task_score": 4.0, "task_score_max": 20, "quality_score": 20.0, "quality_cap_reason": "missing_birch_css_and_visibly_unstyled", "quality_class": "fail" }, { "suite": "publish", "model": "kimi", "model_slug": "kimi", "source_kind": "clean-final", "label": "skill-with-shell-kimi-publication-final", "eval": "implementation-plan", "artifact_path": "results/publish/models/kimi/artifacts/implementation-plan.html", "screenshot_desktop_path": "results/publish/models/kimi/reports/screenshots/implementation-plan-desktop.png", "screenshot_mobile_path": "results/publish/models/kimi/reports/screenshots/implementation-plan-mobile.png", "screenshot_deep_path": "results/publish/models/kimi/reports/screenshots/implementation-plan-deep.png", "screenshot_mobile_deep_path": "results/publish/models/kimi/reports/screenshots/implementation-plan-mobile-deep.png", "artifact_bytes": 50937, "generation_ok": true, "generation_duration_s": 372.779, "input_tokens": 468652, "output_tokens": 19358, "total_tokens": 488010, "billing_tokens": 488010, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 415232, "total_cache_tokens": 415232, "effective_input_tokens": 53420, "display_input_tokens": 468652, "usage_event_count": 15, "tool_calls": 16, "turn_count": 15, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 1, "self_check_failed_runs": 0, "self_check_successful_runs": 1, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 15, "self_check_mode": "run-checker-cli", "self_check_evidence": "ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-kimi-publication-final/implementation-", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "kimi", "model_slug": "kimi", "source_kind": "clean-final", "label": "skill-with-shell-kimi-publication-final", "eval": "benchmark-comparison", "artifact_path": "results/publish/models/kimi/artifacts/benchmark-comparison.html", "screenshot_desktop_path": "results/publish/models/kimi/reports/screenshots/benchmark-comparison-desktop.png", "screenshot_mobile_path": "results/publish/models/kimi/reports/screenshots/benchmark-comparison-mobile.png", "screenshot_deep_path": "results/publish/models/kimi/reports/screenshots/benchmark-comparison-deep.png", "screenshot_mobile_deep_path": "results/publish/models/kimi/reports/screenshots/benchmark-comparison-mobile-deep.png", "artifact_bytes": 51725, "generation_ok": true, "generation_duration_s": 427.336, "input_tokens": 358341, "output_tokens": 15297, "total_tokens": 373638, "billing_tokens": 373638, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 299776, "total_cache_tokens": 299776, "effective_input_tokens": 58565, "display_input_tokens": 358341, "usage_event_count": 14, "tool_calls": 14, "turn_count": 14, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 1, "self_check_failed_runs": 0, "self_check_successful_runs": 1, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 14, "self_check_mode": "run-checker-cli", "self_check_evidence": "ran checker CLI: uv run --with pillow python /home/shaun/source/birch-html/skill/scripts/check_birch_renderings.py --artifact /home/shaun/source/birch-html/eval-runs/skill-with-shell-kimi-publicati", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 1, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 1, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 99, "task_score": 19.8, "task_score_max": 20, "quality_score": 99, "quality_cap_reason": "", "quality_class": "warn" }, { "suite": "publish", "model": "kimi27", "model_slug": "kimi27", "source_kind": "clean-final", "label": "skill-with-shell-kimi27-publication-final", "eval": "numeric-data", "artifact_path": "results/publish/models/kimi27/artifacts/numeric-data.html", "screenshot_desktop_path": "results/publish/models/kimi27/reports/screenshots/numeric-data-desktop.png", "screenshot_mobile_path": "results/publish/models/kimi27/reports/screenshots/numeric-data-mobile.png", "screenshot_deep_path": "results/publish/models/kimi27/reports/screenshots/numeric-data-deep.png", "screenshot_mobile_deep_path": "results/publish/models/kimi27/reports/screenshots/numeric-data-mobile-deep.png", "artifact_bytes": 41967, "generation_ok": true, "generation_duration_s": 210.371, "input_tokens": 1978925, "output_tokens": 17532, "total_tokens": 1996457, "billing_tokens": 1996457, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 1687898, "total_cache_tokens": 1687898, "effective_input_tokens": 291027, "display_input_tokens": 1978925, "usage_event_count": 30, "tool_calls": 32, "turn_count": 30, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 3, "self_check_failed_runs": 0, "self_check_successful_runs": 3, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 30, "self_check_mode": "read-checker,run-checker-cli", "self_check_evidence": "ran checker CLI: cd /home/shaun/source/birch-html && if [ -f skill/scripts/check_birch_renderings.py ]; then uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact eval-runs | read /home/shaun/source/birch-html/skill/scripts/check_birch_renderings.py | ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact /home/shaun/source/birch-html/eval-runs/skill-with-shell-kimi27-p | ran checker CLI: cd /home/shaun/source/birch-html && uv run skill/scripts/finish_birch_html.py eval-runs/skill-with-shell-kimi27-publication-final/numeric-data.html && uv run --with pillow python s", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "kimi27", "model_slug": "kimi27", "source_kind": "clean-final", "label": "skill-with-shell-kimi27-publication-final", "eval": "code-review", "artifact_path": "results/publish/models/kimi27/artifacts/code-review.html", "screenshot_desktop_path": "results/publish/models/kimi27/reports/screenshots/code-review-desktop.png", "screenshot_mobile_path": "results/publish/models/kimi27/reports/screenshots/code-review-mobile.png", "screenshot_deep_path": "results/publish/models/kimi27/reports/screenshots/code-review-deep.png", "screenshot_mobile_deep_path": "results/publish/models/kimi27/reports/screenshots/code-review-mobile-deep.png", "artifact_bytes": 47402, "generation_ok": true, "generation_duration_s": 253.252, "input_tokens": 1509119, "output_tokens": 28034, "total_tokens": 1537153, "billing_tokens": 1537153, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 1144217, "total_cache_tokens": 1144217, "effective_input_tokens": 364902, "display_input_tokens": 1509119, "usage_event_count": 25, "tool_calls": 30, "turn_count": 25, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 2, "self_check_failed_runs": 0, "self_check_successful_runs": 2, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 25, "self_check_mode": "run-checker-cli", "self_check_evidence": "ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-kimi27-publication-final/code-review.html 2> | ran checker CLI: cd /home/shaun/source/birch-html && uv run python skill/scripts/finish_birch_html.py eval-runs/skill-with-shell-kimi27-publication-final/code-review.html && uv run --with pillow py", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "kimi27", "model_slug": "kimi27", "source_kind": "clean-final", "label": "skill-with-shell-kimi27-publication-final", "eval": "module-explainer", "artifact_path": "results/publish/models/kimi27/artifacts/module-explainer.html", "screenshot_desktop_path": "results/publish/models/kimi27/reports/screenshots/module-explainer-desktop.png", "screenshot_mobile_path": "results/publish/models/kimi27/reports/screenshots/module-explainer-mobile.png", "screenshot_deep_path": "results/publish/models/kimi27/reports/screenshots/module-explainer-deep.png", "screenshot_mobile_deep_path": "results/publish/models/kimi27/reports/screenshots/module-explainer-mobile-deep.png", "artifact_bytes": 52748, "generation_ok": true, "generation_duration_s": 136.617, "input_tokens": 582570, "output_tokens": 12473, "total_tokens": 595043, "billing_tokens": 595043, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 334281, "total_cache_tokens": 334281, "effective_input_tokens": 248289, "display_input_tokens": 582570, "usage_event_count": 7, "tool_calls": 14, "turn_count": 7, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 1, "self_check_failed_runs": 0, "self_check_successful_runs": 1, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 7, "self_check_mode": "read-checker,run-checker-cli", "self_check_evidence": "read /home/shaun/source/birch-html/scripts/check_birch_renderings.py | ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-kimi27-publication-final/module-explainer.ht", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "kimi27", "model_slug": "kimi27", "source_kind": "clean-final", "label": "skill-with-shell-kimi27-publication-final", "eval": "implementation-plan", "artifact_path": "results/publish/models/kimi27/artifacts/implementation-plan.html", "screenshot_desktop_path": "results/publish/models/kimi27/reports/screenshots/implementation-plan-desktop.png", "screenshot_mobile_path": "results/publish/models/kimi27/reports/screenshots/implementation-plan-mobile.png", "screenshot_deep_path": "results/publish/models/kimi27/reports/screenshots/implementation-plan-deep.png", "screenshot_mobile_deep_path": "results/publish/models/kimi27/reports/screenshots/implementation-plan-mobile-deep.png", "artifact_bytes": 52277, "generation_ok": true, "generation_duration_s": 72.968, "input_tokens": 487122, "output_tokens": 6684, "total_tokens": 493806, "billing_tokens": 493806, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 332350, "total_cache_tokens": 332350, "effective_input_tokens": 154772, "display_input_tokens": 487122, "usage_event_count": 9, "tool_calls": 9, "turn_count": 9, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 2, "self_check_failed_runs": 1, "self_check_successful_runs": 1, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": true, "assistant_turns_trace": 9, "self_check_mode": "run-checker-cli", "self_check_evidence": "ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-kimi27-publication-final/implementatio | ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact /home/shaun/source/birch-html/eval-runs/skill-with-shell-kimi27-p", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "kimi27", "model_slug": "kimi27", "source_kind": "clean-final", "label": "skill-with-shell-kimi27-publication-final", "eval": "benchmark-comparison", "artifact_path": "results/publish/models/kimi27/artifacts/benchmark-comparison.html", "screenshot_desktop_path": "results/publish/models/kimi27/reports/screenshots/benchmark-comparison-desktop.png", "screenshot_mobile_path": "results/publish/models/kimi27/reports/screenshots/benchmark-comparison-mobile.png", "screenshot_deep_path": "results/publish/models/kimi27/reports/screenshots/benchmark-comparison-deep.png", "screenshot_mobile_deep_path": "results/publish/models/kimi27/reports/screenshots/benchmark-comparison-mobile-deep.png", "artifact_bytes": 59856, "generation_ok": true, "generation_duration_s": 159.927, "input_tokens": 1290293, "output_tokens": 18058, "total_tokens": 1308351, "billing_tokens": 1308351, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 1169139, "total_cache_tokens": 1169139, "effective_input_tokens": 121154, "display_input_tokens": 1290293, "usage_event_count": 16, "tool_calls": 19, "turn_count": 16, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 2, "self_check_failed_runs": 1, "self_check_successful_runs": 1, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": true, "assistant_turns_trace": 16, "self_check_mode": "read-checker,run-checker-cli", "self_check_evidence": "read /home/shaun/source/birch-html/skill/scripts/check_birch_renderings.py | ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python3 skill/scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-kimi27-publication-final/benchmark-co | ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python3 skill/scripts/check_birch_renderings.py --artifact /home/shaun/source/birch-html/eval-runs/skill-with-shell-kimi27-", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "minimax27", "model_slug": "minimax27", "source_kind": "clean-final", "label": "skill-with-shell-minimax27-publication-final", "eval": "numeric-data", "artifact_path": "results/publish/models/minimax27/artifacts/numeric-data.html", "screenshot_desktop_path": "results/publish/models/minimax27/reports/screenshots/numeric-data-desktop.png", "screenshot_mobile_path": "results/publish/models/minimax27/reports/screenshots/numeric-data-mobile.png", "screenshot_deep_path": "results/publish/models/minimax27/reports/screenshots/numeric-data-deep.png", "screenshot_mobile_deep_path": "results/publish/models/minimax27/reports/screenshots/numeric-data-mobile-deep.png", "artifact_bytes": 50838, "generation_ok": false, "generation_duration_s": 160.154, "input_tokens": 87235, "output_tokens": 10902, "total_tokens": 98137, "billing_tokens": 98137, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 116736, "total_cache_tokens": 116736, "effective_input_tokens": 81499, "display_input_tokens": 198235, "usage_event_count": 12, "tool_calls": 9, "turn_count": 10, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 2, "self_check_failed_runs": 1, "self_check_successful_runs": 1, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": true, "assistant_turns_trace": 12, "self_check_mode": "", "self_check_evidence": "", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "minimax27", "model_slug": "minimax27", "source_kind": "clean-final", "label": "skill-with-shell-minimax27-publication-final", "eval": "code-review", "artifact_path": "results/publish/models/minimax27/artifacts/code-review.html", "screenshot_desktop_path": "results/publish/models/minimax27/reports/screenshots/code-review-desktop.png", "screenshot_mobile_path": "results/publish/models/minimax27/reports/screenshots/code-review-mobile.png", "screenshot_deep_path": "results/publish/models/minimax27/reports/screenshots/code-review-deep.png", "screenshot_mobile_deep_path": "results/publish/models/minimax27/reports/screenshots/code-review-mobile-deep.png", "artifact_bytes": 43165, "generation_ok": true, "generation_duration_s": 211.215, "input_tokens": 444148, "output_tokens": 7213, "total_tokens": 451361, "billing_tokens": 451361, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 355328, "total_cache_tokens": 355328, "effective_input_tokens": 88820, "display_input_tokens": 444148, "usage_event_count": 18, "tool_calls": 20, "turn_count": 18, "self_check_attempted": false, "self_check_ran": false, "self_check_succeeded": false, "self_check_runs": 0, "self_check_failed_runs": 0, "self_check_successful_runs": 0, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 18, "self_check_mode": "", "self_check_evidence": "", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "minimax27", "model_slug": "minimax27", "source_kind": "clean-final", "label": "skill-with-shell-minimax27-publication-final", "eval": "module-explainer", "artifact_path": "results/publish/models/minimax27/artifacts/module-explainer.html", "screenshot_desktop_path": "results/publish/models/minimax27/reports/screenshots/module-explainer-desktop.png", "screenshot_mobile_path": "results/publish/models/minimax27/reports/screenshots/module-explainer-mobile.png", "screenshot_deep_path": "results/publish/models/minimax27/reports/screenshots/module-explainer-deep.png", "screenshot_mobile_deep_path": "results/publish/models/minimax27/reports/screenshots/module-explainer-mobile-deep.png", "artifact_bytes": 50511, "generation_ok": false, "generation_duration_s": 183.748, "input_tokens": 185140, "output_tokens": 15068, "total_tokens": 200208, "billing_tokens": 200208, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 232320, "total_cache_tokens": 232320, "effective_input_tokens": 148313, "display_input_tokens": 380633, "usage_event_count": 9, "tool_calls": 9, "turn_count": 5, "self_check_attempted": true, "self_check_ran": false, "self_check_succeeded": false, "self_check_runs": 0, "self_check_failed_runs": 0, "self_check_successful_runs": 0, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 9, "self_check_mode": "read-checker", "self_check_evidence": "read /home/shaun/source/birch-html/scripts/check_birch_renderings.py", "deterministic_failures": 4, "deterministic_warnings": 0, "vlm_failures": 4, "vlm_warnings": 0, "deterministic_failure_units": 1, "deterministic_warning_units": 0, "vlm_failure_units": 1, "vlm_warning_units": 0, "desktop_failures": 1, "desktop_warnings": 0, "mobile_failures": 1, "mobile_warnings": 0, "deep_failures": 1, "deep_warnings": 0, "mobile_deep_failures": 1, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 20.0, "task_score": 4.0, "task_score_max": 20, "quality_score": 20.0, "quality_cap_reason": "missing_birch_css_and_visibly_unstyled", "quality_class": "fail" }, { "suite": "publish", "model": "minimax27", "model_slug": "minimax27", "source_kind": "clean-final", "label": "skill-with-shell-minimax27-publication-final", "eval": "implementation-plan", "artifact_path": "results/publish/models/minimax27/artifacts/implementation-plan.html", "screenshot_desktop_path": "results/publish/models/minimax27/reports/screenshots/implementation-plan-desktop.png", "screenshot_mobile_path": "results/publish/models/minimax27/reports/screenshots/implementation-plan-mobile.png", "screenshot_deep_path": "results/publish/models/minimax27/reports/screenshots/implementation-plan-deep.png", "screenshot_mobile_deep_path": "results/publish/models/minimax27/reports/screenshots/implementation-plan-mobile-deep.png", "artifact_bytes": 21904, "generation_ok": false, "generation_duration_s": 64.763, "input_tokens": 27146, "output_tokens": 4563, "total_tokens": 31709, "billing_tokens": 31709, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 7040, "total_cache_tokens": 7040, "effective_input_tokens": 11494, "display_input_tokens": 18534, "usage_event_count": 3, "tool_calls": 3, "turn_count": 4, "self_check_attempted": false, "self_check_ran": false, "self_check_succeeded": false, "self_check_runs": 0, "self_check_failed_runs": 0, "self_check_successful_runs": 0, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 3, "self_check_mode": "", "self_check_evidence": "", "deterministic_failures": 14, "deterministic_warnings": 4, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 4, "deterministic_warning_units": 1, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 3, "desktop_warnings": 1, "mobile_failures": 4, "mobile_warnings": 1, "deep_failures": 3, "deep_warnings": 1, "mobile_deep_failures": 4, "mobile_deep_warnings": 1, "artifact_present": true, "artifact_score_100": 35.0, "task_score": 7.0, "task_score_max": 20, "quality_score": 35.0, "quality_cap_reason": "missing_birch_css", "quality_class": "fail" }, { "suite": "publish", "model": "minimax27", "model_slug": "minimax27", "source_kind": "clean-final", "label": "skill-with-shell-minimax27-publication-final", "eval": "benchmark-comparison", "artifact_path": "results/publish/models/minimax27/artifacts/benchmark-comparison.html", "screenshot_desktop_path": "results/publish/models/minimax27/reports/screenshots/benchmark-comparison-desktop.png", "screenshot_mobile_path": "results/publish/models/minimax27/reports/screenshots/benchmark-comparison-mobile.png", "screenshot_deep_path": "results/publish/models/minimax27/reports/screenshots/benchmark-comparison-deep.png", "screenshot_mobile_deep_path": "results/publish/models/minimax27/reports/screenshots/benchmark-comparison-mobile-deep.png", "artifact_bytes": 79228, "generation_ok": false, "generation_duration_s": 420.033, "input_tokens": 511926, "output_tokens": 33192, "total_tokens": 545118, "billing_tokens": 545118, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cache_hit_tokens": 129664, "total_cache_tokens": 129664, "effective_input_tokens": 154885, "display_input_tokens": 284549, "usage_event_count": 7, "tool_calls": 14, "turn_count": 13, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 1, "self_check_failed_runs": 0, "self_check_successful_runs": 1, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 7, "self_check_mode": "run-checker-cli", "self_check_evidence": "ran checker CLI: cd /home/shaun/source/birch-html && uv run skill/scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-minimax27-publication-final/benchmark-comparison.html 2>&1 ", "deterministic_failures": 8, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 4, "deterministic_failure_units": 2, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 1, "desktop_failures": 2, "desktop_warnings": 0, "mobile_failures": 2, "mobile_warnings": 0, "deep_failures": 2, "deep_warnings": 0, "mobile_deep_failures": 2, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 35.0, "task_score": 7.0, "task_score_max": 20, "quality_score": 35.0, "quality_cap_reason": "missing_birch_css", "quality_class": "fail" }, { "suite": "publish", "model": "opus47", "model_slug": "opus47", "source_kind": "clean-final", "label": "skill-with-shell-opus47-publication-final", "eval": "numeric-data", "artifact_path": "results/publish/models/opus47/artifacts/numeric-data.html", "screenshot_desktop_path": "results/publish/models/opus47/reports/screenshots/numeric-data-desktop.png", "screenshot_mobile_path": "results/publish/models/opus47/reports/screenshots/numeric-data-mobile.png", "screenshot_deep_path": "results/publish/models/opus47/reports/screenshots/numeric-data-deep.png", "screenshot_mobile_deep_path": "results/publish/models/opus47/reports/screenshots/numeric-data-mobile-deep.png", "artifact_bytes": 45758, "generation_ok": true, "generation_duration_s": 106.088, "input_tokens": 161380, "output_tokens": 8823, "total_tokens": 170203, "billing_tokens": 170203, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 114642, "cache_write_tokens": 25769, "cache_hit_tokens": 0, "total_cache_tokens": 140411, "effective_input_tokens": 20969, "display_input_tokens": 161380, "usage_event_count": 10, "tool_calls": 12, "turn_count": 10, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 2, "self_check_failed_runs": 0, "self_check_successful_runs": 2, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 10, "self_check_mode": "run-checker-cli", "self_check_evidence": "ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-opus47-publication-final/numeric-data. | ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact /home/shaun/source/birch-html/eval-runs/skill-with-shell-opus47-p", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "opus47", "model_slug": "opus47", "source_kind": "clean-final", "label": "skill-with-shell-opus47-publication-final", "eval": "code-review", "artifact_path": "results/publish/models/opus47/artifacts/code-review.html", "screenshot_desktop_path": "results/publish/models/opus47/reports/screenshots/code-review-desktop.png", "screenshot_mobile_path": "results/publish/models/opus47/reports/screenshots/code-review-mobile.png", "screenshot_deep_path": "results/publish/models/opus47/reports/screenshots/code-review-deep.png", "screenshot_mobile_deep_path": "results/publish/models/opus47/reports/screenshots/code-review-mobile-deep.png", "artifact_bytes": 50191, "generation_ok": true, "generation_duration_s": 268.356, "input_tokens": 571314, "output_tokens": 17059, "total_tokens": 588373, "billing_tokens": 588373, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 441950, "cache_write_tokens": 55976, "cache_hit_tokens": 0, "total_cache_tokens": 497926, "effective_input_tokens": 73388, "display_input_tokens": 571314, "usage_event_count": 14, "tool_calls": 18, "turn_count": 14, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 3, "self_check_failed_runs": 0, "self_check_successful_runs": 3, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 14, "self_check_mode": "run-checker-cli", "self_check_evidence": "ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-opus47-publication-final/code-review.h | ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact /home/shaun/source/birch-html/eval-runs/skill-with-shell-opus47-p", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "opus47", "model_slug": "opus47", "source_kind": "clean-final", "label": "skill-with-shell-opus47-publication-final", "eval": "module-explainer", "artifact_path": "results/publish/models/opus47/artifacts/module-explainer.html", "screenshot_desktop_path": "results/publish/models/opus47/reports/screenshots/module-explainer-desktop.png", "screenshot_mobile_path": "results/publish/models/opus47/reports/screenshots/module-explainer-mobile.png", "screenshot_deep_path": "results/publish/models/opus47/reports/screenshots/module-explainer-deep.png", "screenshot_mobile_deep_path": "results/publish/models/opus47/reports/screenshots/module-explainer-mobile-deep.png", "artifact_bytes": 58814, "generation_ok": true, "generation_duration_s": 206.748, "input_tokens": 653611, "output_tokens": 15632, "total_tokens": 669243, "billing_tokens": 669243, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 502232, "cache_write_tokens": 65941, "cache_hit_tokens": 0, "total_cache_tokens": 568173, "effective_input_tokens": 85438, "display_input_tokens": 653611, "usage_event_count": 13, "tool_calls": 19, "turn_count": 13, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 1, "self_check_failed_runs": 0, "self_check_successful_runs": 1, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 13, "self_check_mode": "read-checker,run-checker-cli", "self_check_evidence": "read /home/shaun/source/birch-html/scripts/check_birch_renderings.py | ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-opus47-publication-final/module-explainer.ht", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "opus47", "model_slug": "opus47", "source_kind": "clean-final", "label": "skill-with-shell-opus47-publication-final", "eval": "implementation-plan", "artifact_path": "results/publish/models/opus47/artifacts/implementation-plan.html", "screenshot_desktop_path": "results/publish/models/opus47/reports/screenshots/implementation-plan-desktop.png", "screenshot_mobile_path": "results/publish/models/opus47/reports/screenshots/implementation-plan-mobile.png", "screenshot_deep_path": "results/publish/models/opus47/reports/screenshots/implementation-plan-deep.png", "screenshot_mobile_deep_path": "results/publish/models/opus47/reports/screenshots/implementation-plan-mobile-deep.png", "artifact_bytes": 53012, "generation_ok": true, "generation_duration_s": 141.632, "input_tokens": 206186, "output_tokens": 9414, "total_tokens": 215600, "billing_tokens": 215600, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 160139, "cache_write_tokens": 23940, "cache_hit_tokens": 0, "total_cache_tokens": 184079, "effective_input_tokens": 22107, "display_input_tokens": 206186, "usage_event_count": 11, "tool_calls": 12, "turn_count": 11, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 2, "self_check_failed_runs": 0, "self_check_successful_runs": 2, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 11, "self_check_mode": "run-checker-cli", "self_check_evidence": "ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-opus47-publication-final/implementatio | ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact /home/shaun/source/birch-html/eval-runs/skill-with-shell-opus47-p", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "opus47", "model_slug": "opus47", "source_kind": "clean-final", "label": "skill-with-shell-opus47-publication-final", "eval": "benchmark-comparison", "artifact_path": "results/publish/models/opus47/artifacts/benchmark-comparison.html", "screenshot_desktop_path": "results/publish/models/opus47/reports/screenshots/benchmark-comparison-desktop.png", "screenshot_mobile_path": "results/publish/models/opus47/reports/screenshots/benchmark-comparison-mobile.png", "screenshot_deep_path": "results/publish/models/opus47/reports/screenshots/benchmark-comparison-deep.png", "screenshot_mobile_deep_path": "results/publish/models/opus47/reports/screenshots/benchmark-comparison-mobile-deep.png", "artifact_bytes": 64934, "generation_ok": true, "generation_duration_s": 150.046, "input_tokens": 388331, "output_tokens": 9617, "total_tokens": 397948, "billing_tokens": 397948, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 328368, "cache_write_tokens": 33477, "cache_hit_tokens": 0, "total_cache_tokens": 361845, "effective_input_tokens": 26486, "display_input_tokens": 388331, "usage_event_count": 19, "tool_calls": 22, "turn_count": 19, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 2, "self_check_failed_runs": 0, "self_check_successful_runs": 2, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 19, "self_check_mode": "run-checker-cli", "self_check_evidence": "ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-opus47-publication-final/benchmark-com | ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact /home/shaun/source/birch-html/eval-runs/skill-with-shell-opus47-p", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "sonnet46", "model_slug": "sonnet46", "source_kind": "clean-final", "label": "skill-with-shell-sonnet46-publication-final", "eval": "numeric-data", "artifact_path": "results/publish/models/sonnet46/artifacts/numeric-data.html", "screenshot_desktop_path": "results/publish/models/sonnet46/reports/screenshots/numeric-data-desktop.png", "screenshot_mobile_path": "results/publish/models/sonnet46/reports/screenshots/numeric-data-mobile.png", "screenshot_deep_path": "results/publish/models/sonnet46/reports/screenshots/numeric-data-deep.png", "screenshot_mobile_deep_path": "results/publish/models/sonnet46/reports/screenshots/numeric-data-mobile-deep.png", "artifact_bytes": 52394, "generation_ok": true, "generation_duration_s": 203.959, "input_tokens": 302149, "output_tokens": 14758, "total_tokens": 316907, "billing_tokens": 316907, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 234504, "cache_write_tokens": 38197, "cache_hit_tokens": 0, "total_cache_tokens": 272701, "effective_input_tokens": 29448, "display_input_tokens": 302149, "usage_event_count": 13, "tool_calls": 15, "turn_count": 13, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 2, "self_check_failed_runs": 1, "self_check_successful_runs": 1, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": true, "assistant_turns_trace": 13, "self_check_mode": "run-checker-cli", "self_check_evidence": "ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-sonnet46-publication-final/numeric-dat | ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact /home/shaun/source/birch-html/eval-runs/skill-with-shell-sonnet46", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "sonnet46", "model_slug": "sonnet46", "source_kind": "clean-final", "label": "skill-with-shell-sonnet46-publication-final", "eval": "code-review", "artifact_path": "results/publish/models/sonnet46/artifacts/code-review.html", "screenshot_desktop_path": "results/publish/models/sonnet46/reports/screenshots/code-review-desktop.png", "screenshot_mobile_path": "results/publish/models/sonnet46/reports/screenshots/code-review-mobile.png", "screenshot_deep_path": "results/publish/models/sonnet46/reports/screenshots/code-review-deep.png", "screenshot_mobile_deep_path": "results/publish/models/sonnet46/reports/screenshots/code-review-mobile-deep.png", "artifact_bytes": 57805, "generation_ok": true, "generation_duration_s": 302.047, "input_tokens": 477280, "output_tokens": 18427, "total_tokens": 495707, "billing_tokens": 495707, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 368349, "cache_write_tokens": 44875, "cache_hit_tokens": 0, "total_cache_tokens": 413224, "effective_input_tokens": 64056, "display_input_tokens": 477280, "usage_event_count": 14, "tool_calls": 18, "turn_count": 14, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 2, "self_check_failed_runs": 0, "self_check_successful_runs": 2, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 14, "self_check_mode": "run-checker-cli", "self_check_evidence": "ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-sonnet46-publication-final/code-review | ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact /home/shaun/source/birch-html/eval-runs/skill-with-shell-sonnet46", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "sonnet46", "model_slug": "sonnet46", "source_kind": "clean-final", "label": "skill-with-shell-sonnet46-publication-final", "eval": "module-explainer", "artifact_path": "results/publish/models/sonnet46/artifacts/module-explainer.html", "screenshot_desktop_path": "results/publish/models/sonnet46/reports/screenshots/module-explainer-desktop.png", "screenshot_mobile_path": "results/publish/models/sonnet46/reports/screenshots/module-explainer-mobile.png", "screenshot_deep_path": "results/publish/models/sonnet46/reports/screenshots/module-explainer-deep.png", "screenshot_mobile_deep_path": "results/publish/models/sonnet46/reports/screenshots/module-explainer-mobile-deep.png", "artifact_bytes": 66525, "generation_ok": true, "generation_duration_s": 978.64, "input_tokens": 2649057, "output_tokens": 62243, "total_tokens": 2711300, "billing_tokens": 2711300, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 2413844, "cache_write_tokens": 135163, "cache_hit_tokens": 0, "total_cache_tokens": 2549007, "effective_input_tokens": 100050, "display_input_tokens": 2649057, "usage_event_count": 34, "tool_calls": 38, "turn_count": 34, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 2, "self_check_failed_runs": 1, "self_check_successful_runs": 1, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": true, "assistant_turns_trace": 34, "self_check_mode": "read-checker,run-checker-cli", "self_check_evidence": "read /home/shaun/source/birch-html/scripts/check_birch_renderings.py | ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-sonnet46-publication-final/module-explainer. | ran checker CLI: cd /home/shaun/source/birch-html && uv run skill/scripts/finish_birch_html.py eval-runs/skill-with-shell-sonnet46-publication-final/module-explainer.html && uv run --with pillow py", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "sonnet46", "model_slug": "sonnet46", "source_kind": "clean-final", "label": "skill-with-shell-sonnet46-publication-final", "eval": "implementation-plan", "artifact_path": "results/publish/models/sonnet46/artifacts/implementation-plan.html", "screenshot_desktop_path": "results/publish/models/sonnet46/reports/screenshots/implementation-plan-desktop.png", "screenshot_mobile_path": "results/publish/models/sonnet46/reports/screenshots/implementation-plan-mobile.png", "screenshot_deep_path": "results/publish/models/sonnet46/reports/screenshots/implementation-plan-deep.png", "screenshot_mobile_deep_path": "results/publish/models/sonnet46/reports/screenshots/implementation-plan-mobile-deep.png", "artifact_bytes": 49926, "generation_ok": true, "generation_duration_s": 196.05, "input_tokens": 257093, "output_tokens": 12916, "total_tokens": 270009, "billing_tokens": 270009, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 210864, "cache_write_tokens": 24527, "cache_hit_tokens": 0, "total_cache_tokens": 235391, "effective_input_tokens": 21702, "display_input_tokens": 257093, "usage_event_count": 14, "tool_calls": 15, "turn_count": 14, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 2, "self_check_failed_runs": 0, "self_check_successful_runs": 2, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 14, "self_check_mode": "run-checker-cli", "self_check_evidence": "ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-sonnet46-publication-final/implementat | ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact /home/shaun/source/birch-html/eval-runs/skill-with-shell-sonnet46", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" }, { "suite": "publish", "model": "sonnet46", "model_slug": "sonnet46", "source_kind": "clean-final", "label": "skill-with-shell-sonnet46-publication-final", "eval": "benchmark-comparison", "artifact_path": "results/publish/models/sonnet46/artifacts/benchmark-comparison.html", "screenshot_desktop_path": "results/publish/models/sonnet46/reports/screenshots/benchmark-comparison-desktop.png", "screenshot_mobile_path": "results/publish/models/sonnet46/reports/screenshots/benchmark-comparison-mobile.png", "screenshot_deep_path": "results/publish/models/sonnet46/reports/screenshots/benchmark-comparison-deep.png", "screenshot_mobile_deep_path": "results/publish/models/sonnet46/reports/screenshots/benchmark-comparison-mobile-deep.png", "artifact_bytes": 122208, "generation_ok": true, "generation_duration_s": 623.147, "input_tokens": 1192904, "output_tokens": 48270, "total_tokens": 1241174, "billing_tokens": 1241174, "reasoning_tokens": 0, "tool_use_tokens": 0, "cache_read_tokens": 987803, "cache_write_tokens": 129337, "cache_hit_tokens": 0, "total_cache_tokens": 1117140, "effective_input_tokens": 75764, "display_input_tokens": 1192904, "usage_event_count": 18, "tool_calls": 22, "turn_count": 18, "self_check_attempted": true, "self_check_ran": true, "self_check_succeeded": true, "self_check_runs": 3, "self_check_failed_runs": 0, "self_check_successful_runs": 3, "self_correction_edits": 0, "self_corrected_after_checker": false, "self_correction_verified": false, "assistant_turns_trace": 18, "self_check_mode": "run-checker-cli", "self_check_evidence": "ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact eval-runs/skill-with-shell-sonnet46-publication-final/benchmark-c | ran checker CLI: cd /home/shaun/source/birch-html && uv run --with pillow python skill/scripts/check_birch_renderings.py --artifact /home/shaun/source/birch-html/eval-runs/skill-with-shell-sonnet46", "deterministic_failures": 0, "deterministic_warnings": 0, "vlm_failures": 0, "vlm_warnings": 0, "deterministic_failure_units": 0, "deterministic_warning_units": 0, "vlm_failure_units": 0, "vlm_warning_units": 0, "desktop_failures": 0, "desktop_warnings": 0, "mobile_failures": 0, "mobile_warnings": 0, "deep_failures": 0, "deep_warnings": 0, "mobile_deep_failures": 0, "mobile_deep_warnings": 0, "artifact_present": true, "artifact_score_100": 100.0, "task_score": 20.0, "task_score_max": 20, "quality_score": 100.0, "quality_cap_reason": "", "quality_class": "clean" } ]