Dataset: anime-tags-dev-resolution · revision 20261026-v1 · 1024 selected · silver annotations
A frozen 1,024-image balanced Danbooru-derived selection with incomplete general/character source tags. Exact sample IDs, order, annotations, image hashes/source URLs, pinned profiles and sanitized full predictions are published in the linked reproduction kit. All CAFormer variants use unchanged FP32 weights with AMP FP16; native384 is the original preprocessing control and direct384/512/640 use one resize. PixAI Mexes uses ONNX FP16 at 1008px. One timing pass per pipeline; background GPU applications were not stopped.
Show experiment details
{
"schema_version": 1,
"kind": "public_tagger_comparison",
"exporter_version": "0.3.3",
"title": "Autotagging: PixAI 1.0 and CAFormer resolution variants",
"dataset": {
"id": "anime-tags-dev-resolution",
"revision": "20261026-v1",
"annotation_kind": "silver",
"fingerprint": "6a0e70f23bb08d427f284dbddef21d84b26e63e1b32b3321d939fa174eadd79b",
"manifest_sha256": "93c6b211052055e5e501b81664dd05b770db0c35cf70dab0349586c8ac8e4f68",
"samples_sha256": "ccdbc1e179465f25d8502e32802e74e835bb574283c6d30d8dfa30f2e33ba5e0",
"selected_count": 1024,
"public_reference_url": "https://huggingface.co/spaces/TimeOfAnarchy/vision-bench/blob/d69d975a3d0a07ceae2ceb4bb16b236bb6dc9a92/experiments/autotagging/reproduce/v1/README.md",
"author_description": "A frozen 1,024-image balanced Danbooru-derived selection with incomplete general/character source tags. Exact sample IDs, order, annotations, image hashes/source URLs, pinned profiles and sanitized full predictions are published in the linked reproduction kit. All CAFormer variants use unchanged FP32 weights with AMP FP16; native384 is the original preprocessing control and direct384/512/640 use one resize. PixAI Mexes uses ONNX FP16 at 1008px. One timing pass per pipeline; background GPU applications were not stopped."
},
"author_hardware_note": {
"text": "",
"provenance": "author supplied; not machine-attested"
},
"runs": [
{
"run_id": "full-caformer-native384",
"profile_id": "caformer-native384",
"run_manifest_sha256": "34444b4d062f207fa2c54a68e69ee532084a737c43127749056c6059705e3d9b",
"started_at": "2026-10-26T13:55:40.968193+00:00",
"completed_at": "2026-10-26T13:56:24.083767+00:00",
"model_source": {
"repository": "animetimm/caformer_b36.dbv4-full",
"revision": "aac0699c88553d50eb673b41a81d8222936b22b2",
"precision": "fp32-weights-amp-fp16"
},
"artifact_hashes": {
"config": "7d37124c2cee13d743a8843a42f623d6e626b625e4eea5876cec7fea06f9270f",
"model": "75d656d3f3eec41753cd5553e088ea5d041203bf0e2696d6b4282a998ee43992",
"preprocess": "60c3baca177a4be5513c7bd167c60411393d8af0e0463d6bf2ba3cb43db1a74a",
"vocabulary": "d99cf8cc1293bb7c7671b425d25e5f3688f6937f191ab6957de93acfefdb0e55"
},
"profile_fingerprint": "751ef2da42f804ae1d2bc519f47d772b4eb61417f67b71f44ddd2d1976b1da80",
"metrics_version": 2,
"normalization": "danbooru-nfkc-casefold-v1",
"model_outputs": {
"total": 12476,
"named": 12476,
"unnamed": 0
},
"thresholds": {
"character": 0.47,
"general": 0.39
},
"preprocess": "caformer-b36",
"runtime": {
"platform": "Windows-11-10.0.26200-SP0",
"preprocess_version": "image-taggers-v1",
"preprocess_code_sha256": "b5b034b11d7523ed15bc9814858048e45f0dfc768674765c73d6b404d71904fe",
"python": "3.13.3",
"libraries": {
"vision-bench": "0.3.2",
"numpy": "2.5.2",
"Pillow": "12.3.0"
},
"model": {
"backend": "torch-caformer",
"torch": "2.11.0+cu128",
"timm": "1.0.30",
"cuda": "12.8",
"cudnn": 91900,
"precision": "amp-fp16",
"weights_dtype": "float32",
"strict_state_dict": true,
"tf32": false,
"primary_provider": "TorchCUDA",
"providers": [
"TorchCUDA"
],
"input_shape": [
1,
3,
384,
384
],
"output_name": "logits",
"output_kind": "logits"
},
"environment": {
"collector": "host-v1",
"os": "Windows",
"os_release": "11",
"architecture": "AMD64",
"cpu": "AMD Ryzen 9 9950X3D 16-Core Processor",
"logical_cpus": 32,
"ram_bytes": 66094223360,
"gpu_inventory": {
"status": "observed",
"scope": "visible NVIDIA inventory; not session device binding",
"devices": [
{
"name": "NVIDIA GeForce RTX 3080 Ti",
"vram_mib": 12288,
"driver": "610.88"
}
]
}
}
},
"outcomes": {
"attempted": 1024,
"failed": 0,
"successful": 1024
},
"performance": {
"model_load_ms": 3800.3784000175074,
"warm": {
"count": 1019,
"median_ms": 32.48689998872578,
"p95_ms": 41.71559993410483,
"images_per_second": 30.27748123011294
},
"warmup_excluded_rows": 5,
"gpu_memory": {
"status": "observed",
"method": "windows_pdh_gpu_process_memory",
"scope": "sampled process memory across GPU instances; not an instantaneous allocator peak",
"requested_interval_ms": 250,
"valid_samples": 148,
"valid_span_seconds": 38.28055400005542,
"dedicated_observed_high_water_bytes": 1047351296,
"shared_observed_high_water_bytes": 75497472
}
},
"native_quality": {
"character": {
"sample_count": 1024,
"label_count": 3247,
"threshold": 0.47,
"macro_f1_supported": 0.8738321902412132,
"mean_average_precision_supported": 0.9617438872234706,
"micro_full_truth": {
"true_positive": 609,
"unlabeled_positive": 69,
"false_negative": 1078,
"precision": 0.8982300884955752,
"recall": 0.36099585062240663,
"f1": 0.5150105708245243
},
"micro_representable": {
"true_positive": 609,
"unlabeled_positive": 69,
"false_negative": 87,
"precision": 0.8982300884955752,
"recall": 0.875,
"f1": 0.8864628820960698
},
"coverage": {
"truth_labels": 1293,
"represented_truth_labels": 437,
"positive_occurrences": 1687,
"represented_occurrences": 696,
"oov_occurrences": 991,
"occurrence_recall_ceiling": 0.41256668642560756
}
},
"general": {
"sample_count": 1024,
"label_count": 9225,
"threshold": 0.39,
"macro_f1_supported": 0.3466346320402795,
"mean_average_precision_supported": 0.5356699651731355,
"micro_full_truth": {
"true_positive": 18446,
"unlabeled_positive": 13132,
"false_negative": 12889,
"precision": 0.5841408575590601,
"recall": 0.5886708153821605,
"f1": 0.586397088042217
},
"micro_representable": {
"true_positive": 18446,
"unlabeled_positive": 13132,
"false_negative": 11642,
"precision": 0.5841408575590601,
"recall": 0.6130683328901888,
"f1": 0.5982551162715273
},
"coverage": {
"truth_labels": 4964,
"represented_truth_labels": 4047,
"positive_occurrences": 31335,
"represented_occurrences": 30088,
"oov_occurrences": 1247,
"occurrence_recall_ceiling": 0.9602042444550821
}
}
}
},
{
"run_id": "full-caformer-direct384",
"profile_id": "caformer-direct384",
"run_manifest_sha256": "e933af467dddfeadbf1b5a67cda4096337d887255d1c26b3c911fa1c04e71641",
"started_at": "2026-10-26T13:56:24.669886+00:00",
"completed_at": "2026-10-26T13:57:04.786132+00:00",
"model_source": {
"repository": "animetimm/caformer_b36.dbv4-full",
"revision": "aac0699c88553d50eb673b41a81d8222936b22b2",
"precision": "fp32-weights-amp-fp16"
},
"artifact_hashes": {
"config": "7d37124c2cee13d743a8843a42f623d6e626b625e4eea5876cec7fea06f9270f",
"model": "75d656d3f3eec41753cd5553e088ea5d041203bf0e2696d6b4282a998ee43992",
"preprocess": "60c3baca177a4be5513c7bd167c60411393d8af0e0463d6bf2ba3cb43db1a74a",
"vocabulary": "d99cf8cc1293bb7c7671b425d25e5f3688f6937f191ab6957de93acfefdb0e55"
},
"profile_fingerprint": "19ddaa8e7bbd7c4ffe47198912f989c09afe631a6df4b0aaf380f150f855d346",
"metrics_version": 2,
"normalization": "danbooru-nfkc-casefold-v1",
"model_outputs": {
"total": 12476,
"named": 12476,
"unnamed": 0
},
"thresholds": {
"character": 0.47,
"general": 0.39
},
"preprocess": "caformer-b36-direct-384",
"runtime": {
"platform": "Windows-11-10.0.26200-SP0",
"preprocess_version": "image-taggers-v1",
"preprocess_code_sha256": "b5b034b11d7523ed15bc9814858048e45f0dfc768674765c73d6b404d71904fe",
"python": "3.13.3",
"libraries": {
"vision-bench": "0.3.2",
"numpy": "2.5.2",
"Pillow": "12.3.0"
},
"model": {
"backend": "torch-caformer",
"torch": "2.11.0+cu128",
"timm": "1.0.30",
"cuda": "12.8",
"cudnn": 91900,
"precision": "amp-fp16",
"weights_dtype": "float32",
"strict_state_dict": true,
"tf32": false,
"primary_provider": "TorchCUDA",
"providers": [
"TorchCUDA"
],
"input_shape": [
1,
3,
384,
384
],
"output_name": "logits",
"output_kind": "logits"
},
"environment": {
"collector": "host-v1",
"os": "Windows",
"os_release": "11",
"architecture": "AMD64",
"cpu": "AMD Ryzen 9 9950X3D 16-Core Processor",
"logical_cpus": 32,
"ram_bytes": 66094223360,
"gpu_inventory": {
"status": "observed",
"scope": "visible NVIDIA inventory; not session device binding",
"devices": [
{
"name": "NVIDIA GeForce RTX 3080 Ti",
"vram_mib": 12288,
"driver": "610.88"
}
]
}
}
},
"outcomes": {
"attempted": 1024,
"failed": 0,
"successful": 1024
},
"performance": {
"model_load_ms": 3857.7479000668973,
"warm": {
"count": 1019,
"median_ms": 29.526400030590594,
"p95_ms": 38.45946001820264,
"images_per_second": 33.23805482328708
},
"warmup_excluded_rows": 5,
"gpu_memory": {
"status": "observed",
"method": "windows_pdh_gpu_process_memory",
"scope": "sampled process memory across GPU instances; not an instantaneous allocator peak",
"requested_interval_ms": 250,
"valid_samples": 135,
"valid_span_seconds": 34.816845200024545,
"dedicated_observed_high_water_bytes": 1047351296,
"shared_observed_high_water_bytes": 75497472
}
},
"native_quality": {
"character": {
"sample_count": 1024,
"label_count": 3247,
"threshold": 0.47,
"macro_f1_supported": 0.8681310768439092,
"mean_average_precision_supported": 0.9595734854290986,
"micro_full_truth": {
"true_positive": 604,
"unlabeled_positive": 61,
"false_negative": 1083,
"precision": 0.9082706766917293,
"recall": 0.35803200948429165,
"f1": 0.5136054421768708
},
"micro_representable": {
"true_positive": 604,
"unlabeled_positive": 61,
"false_negative": 92,
"precision": 0.9082706766917293,
"recall": 0.867816091954023,
"f1": 0.887582659808964
},
"coverage": {
"truth_labels": 1293,
"represented_truth_labels": 437,
"positive_occurrences": 1687,
"represented_occurrences": 696,
"oov_occurrences": 991,
"occurrence_recall_ceiling": 0.41256668642560756
}
},
"general": {
"sample_count": 1024,
"label_count": 9225,
"threshold": 0.39,
"macro_f1_supported": 0.3404683056606737,
"mean_average_precision_supported": 0.5311912981944107,
"micro_full_truth": {
"true_positive": 18175,
"unlabeled_positive": 12944,
"false_negative": 13160,
"precision": 0.5840483306018831,
"recall": 0.5800223392372746,
"f1": 0.5820283728824415
},
"micro_representable": {
"true_positive": 18175,
"unlabeled_positive": 12944,
"false_negative": 11913,
"precision": 0.5840483306018831,
"recall": 0.6040614198351503,
"f1": 0.5938863201921348
},
"coverage": {
"truth_labels": 4964,
"represented_truth_labels": 4047,
"positive_occurrences": 31335,
"represented_occurrences": 30088,
"oov_occurrences": 1247,
"occurrence_recall_ceiling": 0.9602042444550821
}
}
}
},
{
"run_id": "full-caformer-direct512",
"profile_id": "caformer-direct512",
"run_manifest_sha256": "f8bf6b73005ea0b841462cb45b40c66d20894d3d0df2276694f5c1f844433921",
"started_at": "2026-10-26T13:57:05.438240+00:00",
"completed_at": "2026-10-26T13:58:00.863931+00:00",
"model_source": {
"repository": "animetimm/caformer_b36.dbv4-full",
"revision": "aac0699c88553d50eb673b41a81d8222936b22b2",
"precision": "fp32-weights-amp-fp16"
},
"artifact_hashes": {
"config": "7d37124c2cee13d743a8843a42f623d6e626b625e4eea5876cec7fea06f9270f",
"model": "75d656d3f3eec41753cd5553e088ea5d041203bf0e2696d6b4282a998ee43992",
"preprocess": "60c3baca177a4be5513c7bd167c60411393d8af0e0463d6bf2ba3cb43db1a74a",
"vocabulary": "d99cf8cc1293bb7c7671b425d25e5f3688f6937f191ab6957de93acfefdb0e55"
},
"profile_fingerprint": "9e123cde661f913225844ac3dc5da93fa4979926be3223b65d3cc03c1a442aa9",
"metrics_version": 2,
"normalization": "danbooru-nfkc-casefold-v1",
"model_outputs": {
"total": 12476,
"named": 12476,
"unnamed": 0
},
"thresholds": {
"character": 0.47,
"general": 0.39
},
"preprocess": "caformer-b36-direct-512",
"runtime": {
"platform": "Windows-11-10.0.26200-SP0",
"preprocess_version": "image-taggers-v1",
"preprocess_code_sha256": "b5b034b11d7523ed15bc9814858048e45f0dfc768674765c73d6b404d71904fe",
"python": "3.13.3",
"libraries": {
"vision-bench": "0.3.2",
"numpy": "2.5.2",
"Pillow": "12.3.0"
},
"model": {
"backend": "torch-caformer",
"torch": "2.11.0+cu128",
"timm": "1.0.30",
"cuda": "12.8",
"cudnn": 91900,
"precision": "amp-fp16",
"weights_dtype": "float32",
"strict_state_dict": true,
"tf32": false,
"primary_provider": "TorchCUDA",
"providers": [
"TorchCUDA"
],
"input_shape": [
1,
3,
512,
512
],
"output_name": "logits",
"output_kind": "logits"
},
"environment": {
"collector": "host-v1",
"os": "Windows",
"os_release": "11",
"architecture": "AMD64",
"cpu": "AMD Ryzen 9 9950X3D 16-Core Processor",
"logical_cpus": 32,
"ram_bytes": 66094223360,
"gpu_inventory": {
"status": "observed",
"scope": "visible NVIDIA inventory; not session device binding",
"devices": [
{
"name": "NVIDIA GeForce RTX 3080 Ti",
"vram_mib": 12288,
"driver": "610.88"
}
]
}
}
},
"outcomes": {
"attempted": 1024,
"failed": 0,
"successful": 1024
},
"performance": {
"model_load_ms": 4221.608600113541,
"warm": {
"count": 1019,
"median_ms": 43.61829999834299,
"p95_ms": 52.20779004739597,
"images_per_second": 22.811237527206245
},
"warmup_excluded_rows": 5,
"gpu_memory": {
"status": "observed",
"method": "windows_pdh_gpu_process_memory",
"scope": "sampled process memory across GPU instances; not an instantaneous allocator peak",
"requested_interval_ms": 250,
"valid_samples": 191,
"valid_span_seconds": 49.41129840002395,
"dedicated_observed_high_water_bytes": 1089294336,
"shared_observed_high_water_bytes": 75497472
}
},
"native_quality": {
"character": {
"sample_count": 1024,
"label_count": 3247,
"threshold": 0.47,
"macro_f1_supported": 0.8583087145245757,
"mean_average_precision_supported": 0.9587627237108675,
"micro_full_truth": {
"true_positive": 601,
"unlabeled_positive": 58,
"false_negative": 1086,
"precision": 0.9119878603945372,
"recall": 0.35625370480142265,
"f1": 0.5123614663256607
},
"micro_representable": {
"true_positive": 601,
"unlabeled_positive": 58,
"false_negative": 95,
"precision": 0.9119878603945372,
"recall": 0.8635057471264368,
"f1": 0.8870848708487085
},
"coverage": {
"truth_labels": 1293,
"represented_truth_labels": 437,
"positive_occurrences": 1687,
"represented_occurrences": 696,
"oov_occurrences": 991,
"occurrence_recall_ceiling": 0.41256668642560756
}
},
"general": {
"sample_count": 1024,
"label_count": 9225,
"threshold": 0.39,
"macro_f1_supported": 0.34022141135879214,
"mean_average_precision_supported": 0.5381873180470901,
"micro_full_truth": {
"true_positive": 17926,
"unlabeled_positive": 12246,
"false_negative": 13409,
"precision": 0.5941270051703567,
"recall": 0.5720759534067337,
"f1": 0.5828930040483197
},
"micro_representable": {
"true_positive": 17926,
"unlabeled_positive": 12246,
"false_negative": 12162,
"precision": 0.5941270051703567,
"recall": 0.5957856952938049,
"f1": 0.5949551941586458
},
"coverage": {
"truth_labels": 4964,
"represented_truth_labels": 4047,
"positive_occurrences": 31335,
"represented_occurrences": 30088,
"oov_occurrences": 1247,
"occurrence_recall_ceiling": 0.9602042444550821
}
}
}
},
{
"run_id": "full-caformer-direct640",
"profile_id": "caformer-direct640",
"run_manifest_sha256": "1f0cd577f99524bb710cd11dfadb35ce3b87260d8f58b431aeff9be186004233",
"started_at": "2026-10-26T13:58:01.534029+00:00",
"completed_at": "2026-10-26T13:59:14.326857+00:00",
"model_source": {
"repository": "animetimm/caformer_b36.dbv4-full",
"revision": "aac0699c88553d50eb673b41a81d8222936b22b2",
"precision": "fp32-weights-amp-fp16"
},
"artifact_hashes": {
"config": "7d37124c2cee13d743a8843a42f623d6e626b625e4eea5876cec7fea06f9270f",
"model": "75d656d3f3eec41753cd5553e088ea5d041203bf0e2696d6b4282a998ee43992",
"preprocess": "60c3baca177a4be5513c7bd167c60411393d8af0e0463d6bf2ba3cb43db1a74a",
"vocabulary": "d99cf8cc1293bb7c7671b425d25e5f3688f6937f191ab6957de93acfefdb0e55"
},
"profile_fingerprint": "26081bdb377eb8c4d67a190b3d9332b6f4ae87944cdb17687a38e603a17852c9",
"metrics_version": 2,
"normalization": "danbooru-nfkc-casefold-v1",
"model_outputs": {
"total": 12476,
"named": 12476,
"unnamed": 0
},
"thresholds": {
"character": 0.47,
"general": 0.39
},
"preprocess": "caformer-b36-direct-640",
"runtime": {
"platform": "Windows-11-10.0.26200-SP0",
"preprocess_version": "image-taggers-v1",
"preprocess_code_sha256": "b5b034b11d7523ed15bc9814858048e45f0dfc768674765c73d6b404d71904fe",
"python": "3.13.3",
"libraries": {
"vision-bench": "0.3.2",
"numpy": "2.5.2",
"Pillow": "12.3.0"
},
"model": {
"backend": "torch-caformer",
"torch": "2.11.0+cu128",
"timm": "1.0.30",
"cuda": "12.8",
"cudnn": 91900,
"precision": "amp-fp16",
"weights_dtype": "float32",
"strict_state_dict": true,
"tf32": false,
"primary_provider": "TorchCUDA",
"providers": [
"TorchCUDA"
],
"input_shape": [
1,
3,
640,
640
],
"output_name": "logits",
"output_kind": "logits"
},
"environment": {
"collector": "host-v1",
"os": "Windows",
"os_release": "11",
"architecture": "AMD64",
"cpu": "AMD Ryzen 9 9950X3D 16-Core Processor",
"logical_cpus": 32,
"ram_bytes": 66094223360,
"gpu_inventory": {
"status": "observed",
"scope": "visible NVIDIA inventory; not session device binding",
"devices": [
{
"name": "NVIDIA GeForce RTX 3080 Ti",
"vram_mib": 12288,
"driver": "610.88"
}
]
}
}
},
"outcomes": {
"attempted": 1024,
"failed": 0,
"successful": 1024
},
"performance": {
"model_load_ms": 4531.719000078738,
"warm": {
"count": 1019,
"median_ms": 60.229400056414306,
"p95_ms": 70.21773006999865,
"images_per_second": 16.4436388284914
},
"warmup_excluded_rows": 5,
"gpu_memory": {
"status": "observed",
"method": "windows_pdh_gpu_process_memory",
"scope": "sampled process memory across GPU instances; not an instantaneous allocator peak",
"requested_interval_ms": 250,
"valid_samples": 260,
"valid_span_seconds": 67.27094149996992,
"dedicated_observed_high_water_bytes": 1168986112,
"shared_observed_high_water_bytes": 75497472
}
},
"native_quality": {
"character": {
"sample_count": 1024,
"label_count": 3247,
"threshold": 0.47,
"macro_f1_supported": 0.8512791521275344,
"mean_average_precision_supported": 0.9581164159327983,
"micro_full_truth": {
"true_positive": 599,
"unlabeled_positive": 59,
"false_negative": 1088,
"precision": 0.9103343465045592,
"recall": 0.35506816834617666,
"f1": 0.5108742004264393
},
"micro_representable": {
"true_positive": 599,
"unlabeled_positive": 59,
"false_negative": 97,
"precision": 0.9103343465045592,
"recall": 0.860632183908046,
"f1": 0.8847858197932054
},
"coverage": {
"truth_labels": 1293,
"represented_truth_labels": 437,
"positive_occurrences": 1687,
"represented_occurrences": 696,
"oov_occurrences": 991,
"occurrence_recall_ceiling": 0.41256668642560756
}
},
"general": {
"sample_count": 1024,
"label_count": 9225,
"threshold": 0.39,
"macro_f1_supported": 0.31692039829322727,
"mean_average_precision_supported": 0.523596162318121,
"micro_full_truth": {
"true_positive": 16490,
"unlabeled_positive": 10422,
"false_negative": 14845,
"precision": 0.6127378121284186,
"recall": 0.5262486037976704,
"f1": 0.5662094185108246
},
"micro_representable": {
"true_positive": 16490,
"unlabeled_positive": 10422,
"false_negative": 13598,
"precision": 0.6127378121284186,
"recall": 0.54805902685456,
"f1": 0.5785964912280702
},
"coverage": {
"truth_labels": 4964,
"represented_truth_labels": 4047,
"positive_occurrences": 31335,
"represented_occurrences": 30088,
"oov_occurrences": 1247,
"occurrence_recall_ceiling": 0.9602042444550821
}
}
}
},
{
"run_id": "full-pixai-v1-mexes",
"profile_id": "pixai-v1.0-mexes-fp16",
"run_manifest_sha256": "adb3b6a44df45083bbc81b9d438a53a418344bb0c0713fe52b65bd039407ffd7",
"started_at": "2026-10-26T13:59:15.343087+00:00",
"completed_at": "2026-10-26T14:03:18.514987+00:00",
"model_source": {
"repository": "Mexes/pixai-tagger-v1.0-onnx-fp32-fp16-int8",
"revision": "0800778563144a0e6fdf41ddadd84aae3cb0dbcf",
"precision": "fp16"
},
"artifact_hashes": {
"model": "cd668e6760bb58ec400ce1b0286e5c9133f83b543caac6a378406b7d56f8c303",
"preprocess": "b6cc89f3dbeee8e97666934d812a6a6bcebe81ae4eff763da9f58f4c52cd6ed4",
"vocabulary": "a9455cbf0a910d4a3890739f2f70bd986278f4594285ef1049121a03896e2a6d"
},
"profile_fingerprint": "ee7f18dd6abb0c55d4ee603dd9583f75c987bf51c13ba8dd1b51fd69a2705204",
"metrics_version": 2,
"normalization": "danbooru-nfkc-casefold-v1",
"model_outputs": {
"total": 30877,
"named": 30877,
"unnamed": 0
},
"thresholds": {
"character": 0.27,
"copyright": 0.24,
"general": 0.17,
"meta": 0.17,
"rating": 0.41,
"style": 0.15
},
"preprocess": "pixai-v1.0",
"runtime": {
"platform": "Windows-11-10.0.26200-SP0",
"preprocess_version": "image-taggers-v1",
"preprocess_code_sha256": "b5b034b11d7523ed15bc9814858048e45f0dfc768674765c73d6b404d71904fe",
"python": "3.13.3",
"libraries": {
"vision-bench": "0.3.2",
"numpy": "2.3.5",
"Pillow": "12.3.0"
},
"model": {
"onnxruntime": "1.23.2",
"primary_provider": "CUDAExecutionProvider",
"providers": [
"CUDAExecutionProvider",
"CPUExecutionProvider"
],
"input_shape": [
"batch",
3,
1008,
1008
],
"output_name": "prediction",
"output_kind": "probabilities",
"cpu_graph_partitions_allowed": true
},
"environment": {
"collector": "host-v1",
"os": "Windows",
"os_release": "11",
"architecture": "AMD64",
"cpu": "AMD Ryzen 9 9950X3D 16-Core Processor",
"logical_cpus": 32,
"ram_bytes": 66094223360,
"gpu_inventory": {
"status": "observed",
"scope": "visible NVIDIA inventory; not session device binding",
"devices": [
{
"name": "NVIDIA GeForce RTX 3080 Ti",
"vram_mib": 12288,
"driver": "610.88"
}
]
}
}
},
"outcomes": {
"attempted": 1024,
"failed": 0,
"successful": 1024
},
"performance": {
"model_load_ms": 1702.8660998912528,
"warm": {
"count": 1019,
"median_ms": 223.63010002300143,
"p95_ms": 247.85691993311048,
"images_per_second": 4.430625449741974
},
"warmup_excluded_rows": 5,
"gpu_memory": {
"status": "observed",
"method": "windows_pdh_gpu_process_memory",
"scope": "sampled process memory across GPU instances; not an instantaneous allocator peak",
"requested_interval_ms": 250,
"valid_samples": 919,
"valid_span_seconds": 239.30311560002156,
"dedicated_observed_high_water_bytes": 6732759040,
"shared_observed_high_water_bytes": 69206016
}
},
"native_quality": {
"character": {
"sample_count": 1024,
"label_count": 8308,
"threshold": 0.27,
"macro_f1_supported": 0.8373607196032751,
"mean_average_precision_supported": 0.9622633626127913,
"micro_full_truth": {
"true_positive": 901,
"unlabeled_positive": 139,
"false_negative": 786,
"precision": 0.8663461538461539,
"recall": 0.5340841730883225,
"f1": 0.6607994132746609
},
"micro_representable": {
"true_positive": 901,
"unlabeled_positive": 139,
"false_negative": 158,
"precision": 0.8663461538461539,
"recall": 0.8508026440037771,
"f1": 0.8585040495474036
},
"coverage": {
"truth_labels": 1293,
"represented_truth_labels": 718,
"positive_occurrences": 1687,
"represented_occurrences": 1059,
"oov_occurrences": 628,
"occurrence_recall_ceiling": 0.6277415530527564
}
},
"general": {
"sample_count": 1024,
"label_count": 15043,
"threshold": 0.17,
"macro_f1_supported": 0.40936909224631984,
"mean_average_precision_supported": 0.5807940229497709,
"micro_full_truth": {
"true_positive": 22643,
"unlabeled_positive": 22561,
"false_negative": 8692,
"precision": 0.5009069993805858,
"recall": 0.7226104994415191,
"f1": 0.5916722193914213
},
"micro_representable": {
"true_positive": 22643,
"unlabeled_positive": 22561,
"false_negative": 8293,
"precision": 0.5009069993805858,
"recall": 0.7319304370312905,
"f1": 0.5947727869713685
},
"coverage": {
"truth_labels": 4964,
"represented_truth_labels": 4604,
"positive_occurrences": 31335,
"represented_occurrences": 30936,
"oov_occurrences": 399,
"occurrence_recall_ceiling": 0.9872666347534705
}
}
}
}
],
"comparison": {
"schema_version": 1,
"dataset_fingerprint": "6a0e70f23bb08d427f284dbddef21d84b26e63e1b32b3321d939fa174eadd79b",
"attempted_per_model": 1024,
"paired_successful": 1024,
"categories": {
"character": {
"label_count": 3190,
"vocabulary_sha256": "2bdded19ba3671bf1a68682e6ece8684baa615027f8b92245404e3a931854b6f",
"score_scope": "common_outputs_only; full_truth metrics are not native full-vocabulary metrics",
"runs": [
{
"run_id": "full-caformer-native384",
"profile_id": "caformer-native384",
"successful": 1024,
"failed": 0,
"metrics": {
"sample_count": 1024,
"label_count": 3190,
"threshold": 0.47,
"macro_f1_supported": 0.8738321902412132,
"mean_average_precision_supported": 0.9617438872234706,
"micro_full_truth": {
"true_positive": 609,
"unlabeled_positive": 63,
"false_negative": 1078,
"precision": 0.90625,
"recall": 0.36099585062240663,
"f1": 0.516320474777448
},
"micro_representable": {
"true_positive": 609,
"unlabeled_positive": 63,
"false_negative": 87,
"precision": 0.90625,
"recall": 0.875,
"f1": 0.8903508771929824
},
"coverage": {
"truth_labels": 1293,
"represented_truth_labels": 437,
"positive_occurrences": 1687,
"represented_occurrences": 696,
"oov_occurrences": 991,
"occurrence_recall_ceiling": 0.41256668642560756
}
}
},
{
"run_id": "full-caformer-direct384",
"profile_id": "caformer-direct384",
"successful": 1024,
"failed": 0,
"metrics": {
"sample_count": 1024,
"label_count": 3190,
"threshold": 0.47,
"macro_f1_supported": 0.8681310768439092,
"mean_average_precision_supported": 0.9595734854290986,
"micro_full_truth": {
"true_positive": 604,
"unlabeled_positive": 52,
"false_negative": 1083,
"precision": 0.9207317073170732,
"recall": 0.35803200948429165,
"f1": 0.5155783183952198
},
"micro_representable": {
"true_positive": 604,
"unlabeled_positive": 52,
"false_negative": 92,
"precision": 0.9207317073170732,
"recall": 0.867816091954023,
"f1": 0.893491124260355
},
"coverage": {
"truth_labels": 1293,
"represented_truth_labels": 437,
"positive_occurrences": 1687,
"represented_occurrences": 696,
"oov_occurrences": 991,
"occurrence_recall_ceiling": 0.41256668642560756
}
}
},
{
"run_id": "full-caformer-direct512",
"profile_id": "caformer-direct512",
"successful": 1024,
"failed": 0,
"metrics": {
"sample_count": 1024,
"label_count": 3190,
"threshold": 0.47,
"macro_f1_supported": 0.8583087145245757,
"mean_average_precision_supported": 0.9587627237108675,
"micro_full_truth": {
"true_positive": 601,
"unlabeled_positive": 51,
"false_negative": 1086,
"precision": 0.9217791411042945,
"recall": 0.35625370480142265,
"f1": 0.5138948268490808
},
"micro_representable": {
"true_positive": 601,
"unlabeled_positive": 51,
"false_negative": 95,
"precision": 0.9217791411042945,
"recall": 0.8635057471264368,
"f1": 0.8916913946587537
},
"coverage": {
"truth_labels": 1293,
"represented_truth_labels": 437,
"positive_occurrences": 1687,
"represented_occurrences": 696,
"oov_occurrences": 991,
"occurrence_recall_ceiling": 0.41256668642560756
}
}
},
{
"run_id": "full-caformer-direct640",
"profile_id": "caformer-direct640",
"successful": 1024,
"failed": 0,
"metrics": {
"sample_count": 1024,
"label_count": 3190,
"threshold": 0.47,
"macro_f1_supported": 0.8512791521275344,
"mean_average_precision_supported": 0.9581164159327983,
"micro_full_truth": {
"true_positive": 599,
"unlabeled_positive": 53,
"false_negative": 1088,
"precision": 0.9187116564417178,
"recall": 0.35506816834617666,
"f1": 0.5121846943138093
},
"micro_representable": {
"true_positive": 599,
"unlabeled_positive": 53,
"false_negative": 97,
"precision": 0.9187116564417178,
"recall": 0.860632183908046,
"f1": 0.8887240356083086
},
"coverage": {
"truth_labels": 1293,
"represented_truth_labels": 437,
"positive_occurrences": 1687,
"represented_occurrences": 696,
"oov_occurrences": 991,
"occurrence_recall_ceiling": 0.41256668642560756
}
}
},
{
"run_id": "full-pixai-v1-mexes",
"profile_id": "pixai-v1.0-mexes-fp16",
"successful": 1024,
"failed": 0,
"metrics": {
"sample_count": 1024,
"label_count": 3190,
"threshold": 0.27,
"macro_f1_supported": 0.902777630466772,
"mean_average_precision_supported": 0.9693199295529857,
"micro_full_truth": {
"true_positive": 634,
"unlabeled_positive": 80,
"false_negative": 1053,
"precision": 0.8879551820728291,
"recall": 0.3758150563129816,
"f1": 0.5281132861307788
},
"micro_representable": {
"true_positive": 634,
"unlabeled_positive": 80,
"false_negative": 62,
"precision": 0.8879551820728291,
"recall": 0.9109195402298851,
"f1": 0.8992907801418439
},
"coverage": {
"truth_labels": 1293,
"represented_truth_labels": 437,
"positive_occurrences": 1687,
"represented_occurrences": 696,
"oov_occurrences": 991,
"occurrence_recall_ceiling": 0.41256668642560756
}
}
}
]
},
"general": {
"label_count": 9039,
"vocabulary_sha256": "87f7bc2f0f8b9f2f61e571ca49c32bf29f7ed03a1704b6bfcdcad2aa54096899",
"score_scope": "common_outputs_only; full_truth metrics are not native full-vocabulary metrics",
"runs": [
{
"run_id": "full-caformer-native384",
"profile_id": "caformer-native384",
"successful": 1024,
"failed": 0,
"metrics": {
"sample_count": 1024,
"label_count": 9039,
"threshold": 0.39,
"macro_f1_supported": 0.34672030545403143,
"mean_average_precision_supported": 0.5357994491684978,
"micro_full_truth": {
"true_positive": 18446,
"unlabeled_positive": 12974,
"false_negative": 12889,
"precision": 0.5870782940802037,
"recall": 0.5886708153821605,
"f1": 0.5878734762170345
},
"micro_representable": {
"true_positive": 18446,
"unlabeled_positive": 12974,
"false_negative": 11640,
"precision": 0.5870782940802037,
"recall": 0.6131090872831217,
"f1": 0.599811400513771
},
"coverage": {
"truth_labels": 4964,
"represented_truth_labels": 4046,
"positive_occurrences": 31335,
"represented_occurrences": 30086,
"oov_occurrences": 1249,
"occurrence_recall_ceiling": 0.960140418062869
}
}
},
{
"run_id": "full-caformer-direct384",
"profile_id": "caformer-direct384",
"successful": 1024,
"failed": 0,
"metrics": {
"sample_count": 1024,
"label_count": 9039,
"threshold": 0.39,
"macro_f1_supported": 0.3405524550194629,
"mean_average_precision_supported": 0.5313207905889828,
"micro_full_truth": {
"true_positive": 18175,
"unlabeled_positive": 12791,
"false_negative": 13160,
"precision": 0.5869340567073564,
"recall": 0.5800223392372746,
"f1": 0.5834577294104428
},
"micro_representable": {
"true_positive": 18175,
"unlabeled_positive": 12791,
"false_negative": 11911,
"precision": 0.5869340567073564,
"recall": 0.6041015754836137,
"f1": 0.5953940902836926
},
"coverage": {
"truth_labels": 4964,
"represented_truth_labels": 4046,
"positive_occurrences": 31335,
"represented_occurrences": 30086,
"oov_occurrences": 1249,
"occurrence_recall_ceiling": 0.960140418062869
}
}
},
{
"run_id": "full-caformer-direct512",
"profile_id": "caformer-direct512",
"successful": 1024,
"failed": 0,
"metrics": {
"sample_count": 1024,
"label_count": 9039,
"threshold": 0.39,
"macro_f1_supported": 0.34030549969575674,
"mean_average_precision_supported": 0.5383181623246207,
"micro_full_truth": {
"true_positive": 17926,
"unlabeled_positive": 12074,
"false_negative": 13409,
"precision": 0.5975333333333334,
"recall": 0.5720759534067337,
"f1": 0.5845275943588489
},
"micro_representable": {
"true_positive": 17926,
"unlabeled_positive": 12074,
"false_negative": 12160,
"precision": 0.5975333333333334,
"recall": 0.5958253008043608,
"f1": 0.5966780947308857
},
"coverage": {
"truth_labels": 4964,
"represented_truth_labels": 4046,
"positive_occurrences": 31335,
"represented_occurrences": 30086,
"oov_occurrences": 1249,
"occurrence_recall_ceiling": 0.960140418062869
}
}
},
{
"run_id": "full-caformer-direct640",
"profile_id": "caformer-direct640",
"successful": 1024,
"failed": 0,
"metrics": {
"sample_count": 1024,
"label_count": 9039,
"threshold": 0.39,
"macro_f1_supported": 0.316998727605707,
"mean_average_precision_supported": 0.5237226567205433,
"micro_full_truth": {
"true_positive": 16490,
"unlabeled_positive": 10277,
"false_negative": 14845,
"precision": 0.6160570852168715,
"recall": 0.5262486037976704,
"f1": 0.5676224570582769
},
"micro_representable": {
"true_positive": 16490,
"unlabeled_positive": 10277,
"false_negative": 13596,
"precision": 0.6160570852168715,
"recall": 0.5480954596822443,
"f1": 0.5800925193041704
},
"coverage": {
"truth_labels": 4964,
"represented_truth_labels": 4046,
"positive_occurrences": 31335,
"represented_occurrences": 30086,
"oov_occurrences": 1249,
"occurrence_recall_ceiling": 0.960140418062869
}
}
},
{
"run_id": "full-pixai-v1-mexes",
"profile_id": "pixai-v1.0-mexes-fp16",
"successful": 1024,
"failed": 0,
"metrics": {
"sample_count": 1024,
"label_count": 9039,
"threshold": 0.17,
"macro_f1_supported": 0.4282590883581636,
"mean_average_precision_supported": 0.5823600102723469,
"micro_full_truth": {
"true_positive": 22362,
"unlabeled_positive": 22126,
"false_negative": 8973,
"precision": 0.5026524006473656,
"recall": 0.7136428913355672,
"f1": 0.5898474077786424
},
"micro_representable": {
"true_positive": 22362,
"unlabeled_positive": 22126,
"false_negative": 7724,
"precision": 0.5026524006473656,
"recall": 0.7432692946885595,
"f1": 0.5997264462144983
},
"coverage": {
"truth_labels": 4964,
"represented_truth_labels": 4046,
"positive_occurrences": 31335,
"represented_occurrences": 30086,
"oov_occurrences": 1249,
"occurrence_recall_ceiling": 0.960140418062869
}
}
}
]
}
},
"excluded_categories": {},
"scope": "category-consistent common vocabulary, paired successful samples"
},
"limitations": [
"Paired quality compares the same successful samples and common category vocabulary. Full-truth recall/F1 also count source positives outside that vocabulary.",
"Native quality uses each model's complete vocabulary and its own successful samples; its denominators differ from the paired comparison.",
"Silver source annotations are incomplete. Unlabeled predictions are not verified false positives; native output categories retain their original semantics.",
"Latency covers decoding, preprocessing and inference, excluding integrity checks and loading. Timing uses each run's own successful warm rows, not the paired quality subset.",
"Timing differences are not model speedups unless hardware, provider, runtime and measurement conditions match. Power settings and competing workloads are not automatically recorded.",
"GPU memory is a sampled WDDM process high water including loading, not an instantaneous allocator peak. GPU inventory does not establish which device a session used.",
"Hardware is captured in new runs only. Missing historical fields remain unknown. CUDA/cuDNN loaded-library versions are not attested by an ONNX Runtime version or driver version.",
"This aggregate report contains no sample identities, images or raw scores. A selection fingerprint verifies identity but does not reconstruct the dataset; publish an authorized selection reference separately."
]
}