Benchmarks: replace alloma-3B/1B with Llama-3.1-8B-Instruct-Uz and Mistral-7B-Instruct-Uz
47ca47a verified | { | |
| "schema_version": 1, | |
| "metric_note": "Accuracy for non-translation tasks; COMET for FLORES+; weighted sum normalized by 0.95.", | |
| "weights": { | |
| "uzlib_slice_full": 0.2, | |
| "tumlu_uzbek_full": 0.2, | |
| "flores_en_uz_full": 0.15, | |
| "news_full": 0.1, | |
| "mmlu_english_full": 0.1, | |
| "mmlu_uz_full": 0.1, | |
| "flores_uz_en_full": 0.05, | |
| "sentiment_binary_full": 0.05 | |
| }, | |
| "labels": { | |
| "uzlib_slice_full": "UzLiB", | |
| "tumlu_uzbek_full": "TUMLU-Uzbek", | |
| "flores_en_uz_full": "FLORES+ en to uz", | |
| "news_full": "News", | |
| "mmlu_english_full": "MMLU English", | |
| "mmlu_uz_full": "MMLU Uzbek", | |
| "flores_uz_en_full": "FLORES+ uz to en", | |
| "sentiment_binary_full": "Sentiment" | |
| }, | |
| "models": { | |
| "NeuronAI-4B": { | |
| "run_id": "qwen35_4b_lora_r64_grouped_ckpt4749_public", | |
| "suite_id": "public_uzbek_full_comet_vllm_v2", | |
| "tasks": { | |
| "uzlib_slice_full": 0.6120365394948952, | |
| "tumlu_uzbek_full": 0.45, | |
| "flores_en_uz_full": 0.8965457885374531, | |
| "news_full": 0.7914612766835104, | |
| "mmlu_english_full": 0.6406494801310355, | |
| "mmlu_uz_full": 0.5700754878222476, | |
| "flores_uz_en_full": 0.8762970898743587, | |
| "sentiment_binary_full": 0.9575 | |
| }, | |
| "invalid_rate": { | |
| "uzlib_slice_full": 0.0, | |
| "tumlu_uzbek_full": 0.0, | |
| "flores_en_uz_full": null, | |
| "news_full": 0.0, | |
| "mmlu_english_full": 0.0, | |
| "mmlu_uz_full": 0.0, | |
| "flores_uz_en_full": null, | |
| "sentiment_binary_full": 0.0 | |
| }, | |
| "normalized_weighted_score": 0.6724185843547309 | |
| }, | |
| "Qwen3.5-4B": { | |
| "run_id": "qwen35_4b_instruct_full_20260817_081728", | |
| "suite_id": "public_uzbek_full_vllm", | |
| "tasks": { | |
| "uzlib_slice_full": 0.4030091348737238, | |
| "tumlu_uzbek_full": 0.4042857142857143, | |
| "flores_en_uz_full": 0.8554783304304908, | |
| "news_full": 0.6734350830153656, | |
| "mmlu_english_full": 0.7266058965959266, | |
| "mmlu_uz_full": 0.5257798034468024, | |
| "flores_uz_en_full": 0.8618190287356948, | |
| "sentiment_binary_full": 0.8482 | |
| }, | |
| "invalid_rate": { | |
| "uzlib_slice_full": 0.0, | |
| "tumlu_uzbek_full": 0.0, | |
| "flores_en_uz_full": null, | |
| "news_full": 0.0, | |
| "mmlu_english_full": 0.0, | |
| "mmlu_uz_full": 0.0, | |
| "flores_uz_en_full": null, | |
| "sentiment_binary_full": 0.0145 | |
| }, | |
| "normalized_weighted_score": 0.5977513148832162 | |
| }, | |
| "alloma-8B": { | |
| "run_id": "alloma_8b_full_public_composite", | |
| "suite_id": "public_uzbek_full_composite_apst", | |
| "source_files": [ | |
| "reports/alloma_card_protocol_full/alloma-8B.json", | |
| "reports/eval_suites/external_public/alloma_8b_uzlib_full_20260615/uzlib_full.json", | |
| "reports/eval_suites/public_full_comet/alloma_8b_strict_missing_uz_metrics/tumlu_uzbek_full_apst.json", | |
| "reports/eval_suites/public_full_comet/alloma_8b_strict_missing_uz_metrics/mmlu_uz_full_apst.json" | |
| ], | |
| "tasks": { | |
| "uzlib_slice_full": 0.4239656098871574, | |
| "tumlu_uzbek_full": 0.20714285714285716, | |
| "flores_en_uz_full": 0.8779, | |
| "news_full": 0.5777, | |
| "mmlu_english_full": 0.5347, | |
| "mmlu_uz_full": 0.40037031761857284, | |
| "flores_uz_en_full": 0.8713, | |
| "sentiment_binary_full": 0.7994 | |
| }, | |
| "invalid_rate": { | |
| "uzlib_slice_full": 0.0338527673293928, | |
| "tumlu_uzbek_full": 0.25857142857142856, | |
| "flores_en_uz_full": null, | |
| "news_full": null, | |
| "mmlu_english_full": null, | |
| "mmlu_uz_full": 0.09051417177040308, | |
| "flores_uz_en_full": null, | |
| "sentiment_binary_full": null | |
| }, | |
| "normalized_weighted_score": 0.5186512896503792 | |
| }, | |
| "Llama-3.1-8B-Instruct-Uz": { | |
| "run_id": "strict_full_llama31_8b_instruct_uz", | |
| "suite_id": "public_uzbek_full_comet_vllm_v2", | |
| "tasks": { | |
| "uzlib_slice_full": 0.31649650725416445, | |
| "tumlu_uzbek_full": 0.32, | |
| "flores_en_uz_full": 0.8666636186669038, | |
| "news_full": 0.603372176961947, | |
| "mmlu_english_full": 0.47578692493946734, | |
| "mmlu_uz_full": 0.38719555618857715, | |
| "flores_uz_en_full": 0.7764609643717557, | |
| "sentiment_binary_full": 0.8259 | |
| }, | |
| "invalid_rate": { | |
| "uzlib_slice_full": 0.0, | |
| "tumlu_uzbek_full": 0.05714285714285714, | |
| "flores_en_uz_full": null, | |
| "news_full": 2.0624935547076414e-05, | |
| "mmlu_english_full": 0.0037031761857285288, | |
| "mmlu_uz_full": 0.009471585244267198, | |
| "flores_uz_en_full": null, | |
| "sentiment_binary_full": 0.0045 | |
| }, | |
| "normalized_weighted_score": 0.5095287981878477 | |
| }, | |
| "Mistral-7B-Instruct-Uz": { | |
| "run_id": "strict_full_mistral7b_instruct_uz", | |
| "suite_id": "public_uzbek_full_comet_vllm_v2", | |
| "tasks": { | |
| "uzlib_slice_full": 0.3277807630306287, | |
| "tumlu_uzbek_full": 0.33714285714285713, | |
| "flores_en_uz_full": 0.8859170079972982, | |
| "news_full": 0.6209343095802825, | |
| "mmlu_english_full": 0.294972226178607, | |
| "mmlu_uz_full": 0.3505910838911836, | |
| "flores_uz_en_full": 0.7826120569790223, | |
| "sentiment_binary_full": 0.8083 | |
| }, | |
| "invalid_rate": { | |
| "uzlib_slice_full": 0.011284255776464266, | |
| "tumlu_uzbek_full": 0.05714285714285714, | |
| "flores_en_uz_full": null, | |
| "news_full": 0.0, | |
| "mmlu_english_full": 0.025708588520153824, | |
| "mmlu_uz_full": 0.01167924797037459, | |
| "flores_uz_en_full": null, | |
| "sentiment_binary_full": 0.0459 | |
| }, | |
| "normalized_weighted_score": 0.4969133053139477 | |
| } | |
| }, | |
| "checkpoints": { | |
| "0.25 epoch": { | |
| "run_id": "qwen35_4b_lora_r64_grouped_ckpt1188_public", | |
| "suite_id": "public_uzbek_full_comet_vllm_v2", | |
| "tasks": { | |
| "uzlib_slice_full": 0.6093498119290703, | |
| "tumlu_uzbek_full": 0.44571428571428573, | |
| "flores_en_uz_full": 0.8898104962171044, | |
| "news_full": 0.772197586882541, | |
| "mmlu_english_full": 0.6312491098134169, | |
| "mmlu_uz_full": 0.5549779233727389, | |
| "flores_uz_en_full": 0.8765152560768583, | |
| "sentiment_binary_full": 0.9377 | |
| }, | |
| "invalid_rate": { | |
| "uzlib_slice_full": 0.0, | |
| "tumlu_uzbek_full": 0.0, | |
| "flores_en_uz_full": null, | |
| "news_full": 0.0, | |
| "mmlu_english_full": 0.0, | |
| "mmlu_uz_full": 0.0, | |
| "flores_uz_en_full": null, | |
| "sentiment_binary_full": 0.0001 | |
| }, | |
| "normalized_weighted_score": 0.6642501250231047 | |
| }, | |
| "0.50 epoch": { | |
| "run_id": "qwen35_4b_lora_r64_grouped_ckpt2376_public", | |
| "suite_id": "public_uzbek_full_comet_vllm_v2", | |
| "tasks": { | |
| "uzlib_slice_full": 0.6012896292315959, | |
| "tumlu_uzbek_full": 0.44571428571428573, | |
| "flores_en_uz_full": 0.8914647964086623, | |
| "news_full": 0.7503764050737342, | |
| "mmlu_english_full": 0.621919954422447, | |
| "mmlu_uz_full": 0.5524854009400371, | |
| "flores_uz_en_full": 0.8756085765367719, | |
| "sentiment_binary_full": 0.9524 | |
| }, | |
| "invalid_rate": { | |
| "uzlib_slice_full": 0.0, | |
| "tumlu_uzbek_full": 0.0, | |
| "flores_en_uz_full": null, | |
| "news_full": 0.0, | |
| "mmlu_english_full": 0.0, | |
| "mmlu_uz_full": 0.0, | |
| "flores_uz_en_full": null, | |
| "sentiment_binary_full": 0.0 | |
| }, | |
| "normalized_weighted_score": 0.6599990603378274 | |
| }, | |
| "0.75 epoch": { | |
| "run_id": "qwen35_4b_lora_r64_grouped_seed42_control_public", | |
| "suite_id": "public_uzbek_full_comet_vllm_v2", | |
| "tasks": { | |
| "uzlib_slice_full": 0.6104245029554003, | |
| "tumlu_uzbek_full": 0.39285714285714285, | |
| "flores_en_uz_full": 0.8960352256901529, | |
| "news_full": 0.8070434154893266, | |
| "mmlu_english_full": 0.640364620424441, | |
| "mmlu_uz_full": 0.5671556758296539, | |
| "flores_uz_en_full": 0.8757696719001098, | |
| "sentiment_binary_full": 0.9514 | |
| }, | |
| "invalid_rate": { | |
| "uzlib_slice_full": 0.0, | |
| "tumlu_uzbek_full": 0.0, | |
| "flores_en_uz_full": null, | |
| "news_full": 0.0, | |
| "mmlu_english_full": 0.0, | |
| "mmlu_uz_full": 0.0, | |
| "flores_uz_en_full": null, | |
| "sentiment_binary_full": 0.0 | |
| }, | |
| "normalized_weighted_score": 0.6609225976688203 | |
| }, | |
| "1.00 epoch": { | |
| "run_id": "qwen35_4b_lora_r64_grouped_ckpt4749_public", | |
| "suite_id": "public_uzbek_full_comet_vllm_v2", | |
| "tasks": { | |
| "uzlib_slice_full": 0.6120365394948952, | |
| "tumlu_uzbek_full": 0.45, | |
| "flores_en_uz_full": 0.8965457885374531, | |
| "news_full": 0.7914612766835104, | |
| "mmlu_english_full": 0.6406494801310355, | |
| "mmlu_uz_full": 0.5700754878222476, | |
| "flores_uz_en_full": 0.8762970898743587, | |
| "sentiment_binary_full": 0.9575 | |
| }, | |
| "invalid_rate": { | |
| "uzlib_slice_full": 0.0, | |
| "tumlu_uzbek_full": 0.0, | |
| "flores_en_uz_full": null, | |
| "news_full": 0.0, | |
| "mmlu_english_full": 0.0, | |
| "mmlu_uz_full": 0.0, | |
| "flores_uz_en_full": null, | |
| "sentiment_binary_full": 0.0 | |
| }, | |
| "normalized_weighted_score": 0.6724185843547309 | |
| } | |
| }, | |
| "tokenizer_fertility": { | |
| "status": "superseded", | |
| "result": "fertility_large_20260825.json", | |
| "note": "Use the corpus-stratified 2026-08-25 evaluation; the old 20k uz-crawl-only result was in-domain." | |
| } | |
| } | |