{
  "snapshot": "2026-09-30",
  "calculation": {
    "heading": "One common-input benchmark, three different outcomes",
    "description": "AILANTA recomputed weighted accuracy from nine published suite counts and checked every figure value against the pinned report. These are reused experiment results, not new model runs.",
    "sourceUrl": "https://raw.githubusercontent.com/instax-dutta/sysone-bench/9447a17fccb524957d1211ecdf46153821ff0810/results/v2/report-20260926/figures/figure-source.json",
    "columns": [
      "Jev 1.13.0",
      "Laya 0.3.11",
      "Qwen2.5 1.5B PCD"
    ],
    "rows": [
      {
        "label": "All nine suites (1,240 decisions)",
        "values": [
          "90.65% (1124/1240)",
          "68.63% (851/1240)",
          "60.48% (750/1240)"
        ]
      },
      {
        "label": "Excluding multilingual intent (1,120 decisions)",
        "values": [
          "89.64%",
          "71.16%",
          "59.64%"
        ]
      },
      {
        "label": "agnews",
        "values": [
          "98.75%",
          "85.00%",
          "91.25%"
        ]
      },
      {
        "label": "banking77_12",
        "values": [
          "94.79%",
          "81.25%",
          "58.33%"
        ]
      },
      {
        "label": "emotion",
        "values": [
          "84.38%",
          "65.63%",
          "41.15%"
        ]
      },
      {
        "label": "guardrails",
        "values": [
          "100.00%",
          "76.04%",
          "29.17%"
        ]
      },
      {
        "label": "mnli",
        "values": [
          "86.67%",
          "55.83%",
          "34.17%"
        ]
      },
      {
        "label": "moderation",
        "values": [
          "94.44%",
          "75.69%",
          "72.92%"
        ]
      },
      {
        "label": "multilingual_intent",
        "values": [
          "100.00%",
          "45.00%",
          "68.33%"
        ]
      },
      {
        "label": "sst5",
        "values": [
          "65.00%",
          "33.33%",
          "52.50%"
        ]
      },
      {
        "label": "triage",
        "values": [
          "93.23%",
          "87.50%",
          "78.13%"
        ]
      }
    ],
    "bars": [
      {
        "label": "jev",
        "value": 90.64516129032259
      },
      {
        "label": "laya",
        "value": 68.62903225806451
      },
      {
        "label": "qwen-pcd",
        "value": 60.483870967741936
      }
    ],
    "barUnit": "% correct",
    "conclusion": "The arithmetic agrees across the source tables. Removing the multilingual suite does not reverse the ranking. The labels came from a single human reviewer correcting an AI draft, with no second independent reviewer; this comparison cannot establish broad deployment reliability or customer-domain calibration."
  },
  "sources": [
    {
      "url": "https://raw.githubusercontent.com/instax-dutta/sysone-bench/9447a17fccb524957d1211ecdf46153821ff0810/results/v2/report-20260926/report.json",
      "metadata": {
        "sha": "9447a17fccb524957d1211ecdf46153821ff0810",
        "path": "results/v2/report-20260926/report.json",
        "sha256": "4f2525efce1b3b18a65406e2a7910afbc230bcc1de0d9a77ec1156a9f1c35a82"
      }
    },
    {
      "url": "https://raw.githubusercontent.com/instax-dutta/sysone-bench/9447a17fccb524957d1211ecdf46153821ff0810/results/v2/report-20260926/figures/data/accuracy.csv",
      "metadata": {
        "sha": "9447a17fccb524957d1211ecdf46153821ff0810",
        "path": "results/v2/report-20260926/figures/data/accuracy.csv",
        "sha256": "16252f9a20f9d42130a057315cf6e2af0f8562d246b71c22e3037206808f4805"
      }
    },
    {
      "url": "https://raw.githubusercontent.com/instax-dutta/sysone-bench/9447a17fccb524957d1211ecdf46153821ff0810/results/v2/report-20260926/figures/data/calibration.csv",
      "metadata": {
        "sha": "9447a17fccb524957d1211ecdf46153821ff0810",
        "path": "results/v2/report-20260926/figures/data/calibration.csv",
        "sha256": "05aa42e78f40957a334309da8b7c9a1bbca1cfca8039f3a09e276fd798b64ab6"
      }
    },
    {
      "url": "https://raw.githubusercontent.com/instax-dutta/sysone-bench/9447a17fccb524957d1211ecdf46153821ff0810/results/v2/report-20260926/figures/figure-source.json",
      "metadata": {
        "sha": "9447a17fccb524957d1211ecdf46153821ff0810",
        "path": "results/v2/report-20260926/figures/figure-source.json",
        "sha256": "4b263b8e2cc08d3d5d52fe25b7ac9fc99d7dd2952bca297b4b6c9a4dc6d0af78"
      }
    }
  ]
}