{
  "runDate": "2026-10-01",
  "hardware": "Tesla T4",
  "layaVersion": "0.3.22",
  "modelId": "muhammadnoman76/jev-urdu",
  "metrics": [
    {
      "benchmark": "Laya-Urdu test",
      "prompt": "Urdu",
      "model": "laya (English)",
      "decisions": 751,
      "accuracy": 0.4047936085219707,
      "macro_f1": null,
      "log_loss": 1.1808327077964604,
      "brier": 0.6876209965379495,
      "ece": 0.15981984021304926,
      "confidence": 0.5339639147802929,
      "majority_class": 0.45139813581890814
    },
    {
      "benchmark": "Laya-Urdu test",
      "prompt": "Urdu",
      "model": "laya-multilingual",
      "decisions": 751,
      "accuracy": 0.6391478029294274,
      "macro_f1": null,
      "log_loss": 1.060689715291959,
      "brier": 0.5100731649001332,
      "ece": 0.16290905459387486,
      "confidence": 0.8020568575233024,
      "majority_class": 0.45139813581890814
    },
    {
      "benchmark": "Laya-Urdu test",
      "prompt": "Urdu",
      "model": "laya (routed)",
      "decisions": 751,
      "accuracy": 0.6378162450066578,
      "macro_f1": null,
      "log_loss": 1.061024325387014,
      "brier": 0.5099294265379494,
      "ece": 0.16114141145139813,
      "confidence": 0.798957656458056,
      "majority_class": 0.45139813581890814
    },
    {
      "benchmark": "Laya-Urdu test",
      "prompt": "Urdu",
      "model": "jev-urdu",
      "decisions": 751,
      "accuracy": 1.0,
      "macro_f1": null,
      "log_loss": -0.0,
      "brier": 0.0,
      "ece": 0.0,
      "confidence": 1.0,
      "majority_class": 0.45139813581890814
    },
    {
      "benchmark": "Laya-Urdu test, unseen templates",
      "prompt": "Urdu",
      "model": "laya (English)",
      "decisions": 601,
      "accuracy": 0.3643926788685524,
      "macro_f1": null,
      "log_loss": 1.2791970303012923,
      "brier": 0.7195111704825291,
      "ece": 0.1527695507487521,
      "confidence": 0.5100128119800332,
      "majority_class": 0.44093178036605657
    },
    {
      "benchmark": "Laya-Urdu test, unseen templates",
      "prompt": "Urdu",
      "model": "laya-multilingual",
      "decisions": 601,
      "accuracy": 0.6522462562396006,
      "macro_f1": null,
      "log_loss": 1.0594671493441996,
      "brier": 0.48855896071547417,
      "ece": 0.13482163061564062,
      "confidence": 0.7870678868552413,
      "majority_class": 0.44093178036605657
    },
    {
      "benchmark": "Laya-Urdu test, unseen templates",
      "prompt": "Urdu",
      "model": "laya (routed)",
      "decisions": 601,
      "accuracy": 0.6505823627287853,
      "macro_f1": null,
      "log_loss": 1.059885272774127,
      "brier": 0.4883793475540766,
      "ece": 0.13261281198003327,
      "confidence": 0.7831951747088187,
      "majority_class": 0.44093178036605657
    },
    {
      "benchmark": "Laya-Urdu test, unseen templates",
      "prompt": "Urdu",
      "model": "jev-urdu",
      "decisions": 601,
      "accuracy": 1.0,
      "macro_f1": null,
      "log_loss": -0.0,
      "brier": 0.0,
      "ece": 0.0,
      "confidence": 1.0,
      "majority_class": 0.44093178036605657
    },
    {
      "benchmark": "XNLI-ur",
      "prompt": "Urdu",
      "model": "laya (English)",
      "decisions": 5010,
      "accuracy": 0.3461077844311377,
      "macro_f1": 0.31803296956771515,
      "log_loss": 1.1041822236503773,
      "brier": 0.6697860066307386,
      "ece": 0.024323373253493014,
      "confidence": 0.3704311576846307,
      "majority_class": 0.3333333333333333
    },
    {
      "benchmark": "XNLI-ur",
      "prompt": "Urdu",
      "model": "laya-multilingual",
      "decisions": 5010,
      "accuracy": 0.4608782435129741,
      "macro_f1": 0.4580603154780459,
      "log_loss": 1.3552192118365238,
      "brier": 0.7265916283073852,
      "ece": 0.24792155688622752,
      "confidence": 0.7087998003992017,
      "majority_class": 0.3333333333333333
    },
    {
      "benchmark": "XNLI-ur",
      "prompt": "Urdu",
      "model": "laya (routed)",
      "decisions": 5010,
      "accuracy": 0.4606786427145709,
      "macro_f1": 0.4578066267042531,
      "log_loss": 1.3553055399197023,
      "brier": 0.7266427794011976,
      "ece": 0.24762922155688621,
      "confidence": 0.7083078642714571,
      "majority_class": 0.3333333333333333
    },
    {
      "benchmark": "XNLI-ur",
      "prompt": "Urdu",
      "model": "jev-urdu",
      "decisions": 5010,
      "accuracy": 0.5900199600798404,
      "macro_f1": 0.5892547394900737,
      "log_loss": 3.4041305249110976,
      "brier": 0.793537204744511,
      "ece": 0.39312239520958087,
      "confidence": 0.9828738123752495,
      "majority_class": 0.3333333333333333
    },
    {
      "benchmark": "MASSIVE ur-PK",
      "prompt": "Urdu",
      "model": "laya (English)",
      "decisions": 2974,
      "accuracy": 0.21654337592468056,
      "macro_f1": 0.19268119625011423,
      "log_loss": 3.0056735903582372,
      "brier": 0.925769009667115,
      "ece": 0.14494519166106257,
      "confidence": 0.360072831203766,
      "majority_class": 0.1351714862138534
    },
    {
      "benchmark": "MASSIVE ur-PK",
      "prompt": "Urdu",
      "model": "laya-multilingual",
      "decisions": 2974,
      "accuracy": 0.4835238735709482,
      "macro_f1": 0.4662274596495123,
      "log_loss": 1.9361674689450348,
      "brier": 0.6871877135070612,
      "ece": 0.18025901143241427,
      "confidence": 0.6637828850033625,
      "majority_class": 0.1351714862138534
    },
    {
      "benchmark": "MASSIVE ur-PK",
      "prompt": "Urdu",
      "model": "laya (routed)",
      "decisions": 2974,
      "accuracy": 0.4835238735709482,
      "macro_f1": 0.46605417820602757,
      "log_loss": 1.9360844284187124,
      "brier": 0.6870838632582381,
      "ece": 0.1801401479488904,
      "confidence": 0.6636640215198386,
      "majority_class": 0.1351714862138534
    },
    {
      "benchmark": "MASSIVE ur-PK",
      "prompt": "Urdu",
      "model": "jev-urdu",
      "decisions": 2974,
      "accuracy": 0.4327505043712172,
      "macro_f1": 0.4359245455249651,
      "log_loss": 4.293417570595762,
      "brier": 1.0089685077168797,
      "ece": 0.4760048083389376,
      "confidence": 0.9087553127101548,
      "majority_class": 0.1351714862138534
    },
    {
      "benchmark": "Roman-Urdu sentiment",
      "prompt": "Urdu",
      "model": "laya (English)",
      "decisions": 2048,
      "accuracy": 0.40966796875,
      "macro_f1": 0.2524744301620891,
      "log_loss": 1.0857578634068639,
      "brier": 0.6579363164111328,
      "ece": 0.03860629882812499,
      "confidence": 0.371061669921875,
      "majority_class": 0.44140625
    },
    {
      "benchmark": "Roman-Urdu sentiment",
      "prompt": "Urdu",
      "model": "laya-multilingual",
      "decisions": 2048,
      "accuracy": 0.4404296875,
      "macro_f1": 0.40434830474784816,
      "log_loss": 1.3027094229054903,
      "brier": 0.7313294063183594,
      "ece": 0.2112923828125,
      "confidence": 0.65121103515625,
      "majority_class": 0.44140625
    },
    {
      "benchmark": "Roman-Urdu sentiment",
      "prompt": "Urdu",
      "model": "laya (routed)",
      "decisions": 2048,
      "accuracy": 0.42822265625,
      "macro_f1": 0.313504269179598,
      "log_loss": 1.124182949154887,
      "brier": 0.6659173791113282,
      "ece": 0.07313857421874999,
      "confidence": 0.42034140625000005,
      "majority_class": 0.44140625
    },
    {
      "benchmark": "Roman-Urdu sentiment",
      "prompt": "Urdu",
      "model": "jev-urdu",
      "decisions": 2048,
      "accuracy": 0.47216796875,
      "macro_f1": 0.3839911580420255,
      "log_loss": 4.6999525447182915,
      "brier": 1.0446586475,
      "ece": 0.5197908203124999,
      "confidence": 0.99137138671875,
      "majority_class": 0.44140625
    },
    {
      "benchmark": "XNLI-ur",
      "prompt": "English",
      "model": "laya (English)",
      "decisions": 5010,
      "accuracy": 0.36367265469061877,
      "macro_f1": 0.32220467933915087,
      "log_loss": 1.3928108832209156,
      "brier": 0.8239830435968063,
      "ece": 0.2896666866267465,
      "confidence": 0.6533393413173653,
      "majority_class": 0.3333333333333333
    },
    {
      "benchmark": "XNLI-ur",
      "prompt": "English",
      "model": "laya-multilingual",
      "decisions": 5010,
      "accuracy": 0.6311377245508982,
      "macro_f1": 0.6267208468619827,
      "log_loss": 1.530467280470804,
      "brier": 0.6265925488782436,
      "ece": 0.27330291417165664,
      "confidence": 0.904014730538922,
      "majority_class": 0.3333333333333333
    },
    {
      "benchmark": "XNLI-ur",
      "prompt": "English",
      "model": "laya (routed)",
      "decisions": 5010,
      "accuracy": 0.6301397205588822,
      "macro_f1": 0.6256713737688273,
      "log_loss": 1.5307182813393954,
      "brier": 0.6275275970878244,
      "ece": 0.27387339321357296,
      "confidence": 0.9035872055888224,
      "majority_class": 0.3333333333333333
    },
    {
      "benchmark": "XNLI-ur",
      "prompt": "English",
      "model": "jev-urdu",
      "decisions": 5010,
      "accuracy": 0.6315369261477046,
      "macro_f1": 0.6287523971803266,
      "log_loss": 3.0505972674108692,
      "brier": 0.7136933682175649,
      "ece": 0.3501848702594812,
      "confidence": 0.9817217964071856,
      "majority_class": 0.3333333333333333
    },
    {
      "benchmark": "MASSIVE ur-PK",
      "prompt": "English",
      "model": "laya (English)",
      "decisions": 2974,
      "accuracy": 0.07094821788836583,
      "macro_f1": 0.06604860053591162,
      "log_loss": 5.625268982817806,
      "brier": 1.48418020998655,
      "ece": 0.6707088769334231,
      "confidence": 0.7416570948217889,
      "majority_class": 0.1351714862138534
    },
    {
      "benchmark": "MASSIVE ur-PK",
      "prompt": "English",
      "model": "laya-multilingual",
      "decisions": 2974,
      "accuracy": 0.3201075991930061,
      "macro_f1": 0.33119968611183737,
      "log_loss": 3.6070592576087197,
      "brier": 1.0790946299024882,
      "ece": 0.4676778076664424,
      "confidence": 0.7877854068594483,
      "majority_class": 0.1351714862138534
    },
    {
      "benchmark": "MASSIVE ur-PK",
      "prompt": "English",
      "model": "laya (routed)",
      "decisions": 2974,
      "accuracy": 0.3201075991930061,
      "macro_f1": 0.33114734975832133,
      "log_loss": 3.608024899945605,
      "brier": 1.0792325736381978,
      "ece": 0.4677904841963686,
      "confidence": 0.7878980833893745,
      "majority_class": 0.1351714862138534
    },
    {
      "benchmark": "MASSIVE ur-PK",
      "prompt": "English",
      "model": "jev-urdu",
      "decisions": 2974,
      "accuracy": 0.3406186953597848,
      "macro_f1": 0.3715069533712043,
      "log_loss": 5.7152896242017395,
      "brier": 1.2752123175521184,
      "ece": 0.6289200739744452,
      "confidence": 0.9693609616677875,
      "majority_class": 0.1351714862138534
    },
    {
      "benchmark": "Roman-Urdu sentiment",
      "prompt": "English",
      "model": "laya (English)",
      "decisions": 2048,
      "accuracy": 0.4609375,
      "macro_f1": 0.3541242377147849,
      "log_loss": 1.2599296921087655,
      "brier": 0.7301918913623048,
      "ece": 0.243913232421875,
      "confidence": 0.7048507324218749,
      "majority_class": 0.44140625
    },
    {
      "benchmark": "Roman-Urdu sentiment",
      "prompt": "English",
      "model": "laya-multilingual",
      "decisions": 2048,
      "accuracy": 0.49560546875,
      "macro_f1": 0.4385358387911855,
      "log_loss": 1.790084939227734,
      "brier": 0.8101218942480468,
      "ece": 0.33745400390624997,
      "confidence": 0.8330594726562499,
      "majority_class": 0.44140625
    },
    {
      "benchmark": "Roman-Urdu sentiment",
      "prompt": "English",
      "model": "laya (routed)",
      "decisions": 2048,
      "accuracy": 0.46923828125,
      "macro_f1": 0.380974459267932,
      "log_loss": 1.4022011279897073,
      "brier": 0.753891855258789,
      "ece": 0.26655078125,
      "confidence": 0.7357890625,
      "majority_class": 0.44140625
    },
    {
      "benchmark": "Roman-Urdu sentiment",
      "prompt": "English",
      "model": "jev-urdu",
      "decisions": 2048,
      "accuracy": 0.505859375,
      "macro_f1": 0.4210698657736751,
      "log_loss": 4.207796472052452,
      "brier": 0.9615223229394531,
      "ece": 0.47505717773437495,
      "confidence": 0.9809165527343751,
      "majority_class": 0.44140625
    }
  ],
  "deltas": [
    {
      "benchmark": "Laya-Urdu test",
      "prompt": "Urdu",
      "jev-urdu minus": "laya-multilingual",
      "accuracy Δ": 0.3608521970705726,
      "95% CI low": 0.32756324900133155,
      "95% CI high": 0.3955059920106523,
      "significant": "yes"
    },
    {
      "benchmark": "Laya-Urdu test",
      "prompt": "Urdu",
      "jev-urdu minus": "laya (routed)",
      "accuracy Δ": 0.3621837549933422,
      "95% CI low": 0.3288948069241012,
      "95% CI high": 0.3968042609853529,
      "significant": "yes"
    },
    {
      "benchmark": "Laya-Urdu test",
      "prompt": "Urdu",
      "jev-urdu minus": "laya (English)",
      "accuracy Δ": 0.5952063914780293,
      "95% CI low": 0.5605525965379494,
      "95% CI high": 0.6284953395472703,
      "significant": "yes"
    },
    {
      "benchmark": "Laya-Urdu test, unseen templates",
      "prompt": "Urdu",
      "jev-urdu minus": "laya-multilingual",
      "accuracy Δ": 0.34775374376039936,
      "95% CI low": 0.30948419301164726,
      "95% CI high": 0.3843594009983361,
      "significant": "yes"
    },
    {
      "benchmark": "Laya-Urdu test, unseen templates",
      "prompt": "Urdu",
      "jev-urdu minus": "laya (routed)",
      "accuracy Δ": 0.34941763727121466,
      "95% CI low": 0.30948419301164726,
      "95% CI high": 0.3860232945091514,
      "significant": "yes"
    },
    {
      "benchmark": "Laya-Urdu test, unseen templates",
      "prompt": "Urdu",
      "jev-urdu minus": "laya (English)",
      "accuracy Δ": 0.6356073211314476,
      "95% CI low": 0.5973377703826955,
      "95% CI high": 0.6738768718801996,
      "significant": "yes"
    },
    {
      "benchmark": "XNLI-ur",
      "prompt": "Urdu",
      "jev-urdu minus": "laya-multilingual",
      "accuracy Δ": 0.12914171656686627,
      "95% CI low": 0.1121756487025948,
      "95% CI high": 0.14590818363273453,
      "significant": "yes"
    },
    {
      "benchmark": "XNLI-ur",
      "prompt": "Urdu",
      "jev-urdu minus": "laya (routed)",
      "accuracy Δ": 0.12934131736526946,
      "95% CI low": 0.11237025948103792,
      "95% CI high": 0.14591317365269457,
      "significant": "yes"
    },
    {
      "benchmark": "XNLI-ur",
      "prompt": "Urdu",
      "jev-urdu minus": "laya (English)",
      "accuracy Δ": 0.2439121756487026,
      "95% CI low": 0.22395209580838324,
      "95% CI high": 0.26248003992015967,
      "significant": "yes"
    },
    {
      "benchmark": "MASSIVE ur-PK",
      "prompt": "Urdu",
      "jev-urdu minus": "laya-multilingual",
      "accuracy Δ": -0.050773369199731,
      "95% CI low": -0.06624915938130464,
      "95% CI high": -0.03665097511768662,
      "significant": "yes"
    },
    {
      "benchmark": "MASSIVE ur-PK",
      "prompt": "Urdu",
      "jev-urdu minus": "laya (routed)",
      "accuracy Δ": -0.050773369199731,
      "95% CI low": -0.06624915938130464,
      "95% CI high": -0.03665097511768662,
      "significant": "yes"
    },
    {
      "benchmark": "MASSIVE ur-PK",
      "prompt": "Urdu",
      "jev-urdu minus": "laya (English)",
      "accuracy Δ": 0.21620712844653664,
      "95% CI low": 0.19737726967047747,
      "95% CI high": 0.23538164088769328,
      "significant": "yes"
    },
    {
      "benchmark": "Roman-Urdu sentiment",
      "prompt": "Urdu",
      "jev-urdu minus": "laya-multilingual",
      "accuracy Δ": 0.03173828125,
      "95% CI low": 0.00732421875,
      "95% CI high": 0.05567626953124993,
      "significant": "yes"
    },
    {
      "benchmark": "Roman-Urdu sentiment",
      "prompt": "Urdu",
      "jev-urdu minus": "laya (routed)",
      "accuracy Δ": 0.0439453125,
      "95% CI low": 0.02001953125,
      "95% CI high": 0.0693359375,
      "significant": "yes"
    },
    {
      "benchmark": "Roman-Urdu sentiment",
      "prompt": "Urdu",
      "jev-urdu minus": "laya (English)",
      "accuracy Δ": 0.0625,
      "95% CI low": 0.0380859375,
      "95% CI high": 0.0869140625,
      "significant": "yes"
    },
    {
      "benchmark": "XNLI-ur",
      "prompt": "English",
      "jev-urdu minus": "laya-multilingual",
      "accuracy Δ": 0.0003992015968063872,
      "95% CI low": -0.010978043912175649,
      "95% CI high": 0.01157684630738523,
      "significant": "no"
    },
    {
      "benchmark": "XNLI-ur",
      "prompt": "English",
      "jev-urdu minus": "laya (routed)",
      "accuracy Δ": 0.0013972055888223553,
      "95% CI low": -0.00998003992015968,
      "95% CI high": 0.012375249500998005,
      "significant": "no"
    },
    {
      "benchmark": "XNLI-ur",
      "prompt": "English",
      "jev-urdu minus": "laya (English)",
      "accuracy Δ": 0.2678642714570858,
      "95% CI low": 0.2504940119760479,
      "95% CI high": 0.28483033932135726,
      "significant": "yes"
    },
    {
      "benchmark": "MASSIVE ur-PK",
      "prompt": "English",
      "jev-urdu minus": "laya-multilingual",
      "accuracy Δ": 0.02051109616677875,
      "95% CI low": 0.005379959650302623,
      "95% CI high": 0.03496973772696705,
      "significant": "yes"
    },
    {
      "benchmark": "MASSIVE ur-PK",
      "prompt": "English",
      "jev-urdu minus": "laya (routed)",
      "accuracy Δ": 0.02051109616677875,
      "95% CI low": 0.005379959650302623,
      "95% CI high": 0.03496973772696705,
      "significant": "yes"
    },
    {
      "benchmark": "MASSIVE ur-PK",
      "prompt": "English",
      "jev-urdu minus": "laya (English)",
      "accuracy Δ": 0.269670477471419,
      "95% CI low": 0.25050437121721586,
      "95% CI high": 0.2898453261600538,
      "significant": "yes"
    },
    {
      "benchmark": "Roman-Urdu sentiment",
      "prompt": "English",
      "jev-urdu minus": "laya-multilingual",
      "accuracy Δ": 0.01025390625,
      "95% CI low": -0.01171875,
      "95% CI high": 0.03370361328124993,
      "significant": "no"
    },
    {
      "benchmark": "Roman-Urdu sentiment",
      "prompt": "English",
      "jev-urdu minus": "laya (routed)",
      "accuracy Δ": 0.03662109375,
      "95% CI low": 0.01416015625,
      "95% CI high": 0.05810546875,
      "significant": "yes"
    },
    {
      "benchmark": "Roman-Urdu sentiment",
      "prompt": "English",
      "jev-urdu minus": "laya (English)",
      "accuracy Δ": 0.044921875,
      "95% CI low": 0.02197265625,
      "95% CI high": 0.06689453125,
      "significant": "yes"
    }
  ],
  "headToHead": [
    {
      "prompt": "Urdu",
      "benchmark": "Laya-Urdu test",
      "decisions": 232,
      "scores": {
        "laya (English)": 0.397,
        "laya-multilingual": 0.634,
        "laya (routed)": 0.634,
        "jev (TypeSafe)": 0.823,
        "jev-urdu": 1.0,
        "majority class": 0.448
      }
    },
    {
      "prompt": "Urdu",
      "benchmark": "XNLI-ur",
      "decisions": 100,
      "scores": {
        "laya (English)": 0.3,
        "laya-multilingual": 0.42,
        "laya (routed)": 0.42,
        "jev (TypeSafe)": 0.7,
        "jev-urdu": 0.62,
        "majority class": 0.26
      }
    },
    {
      "prompt": "Urdu",
      "benchmark": "MASSIVE ur-PK",
      "decisions": 100,
      "scores": {
        "laya (English)": 0.21,
        "laya-multilingual": 0.4,
        "laya (routed)": 0.4,
        "jev (TypeSafe)": 0.79,
        "jev-urdu": 0.31,
        "majority class": 0.22
      }
    },
    {
      "prompt": "Urdu",
      "benchmark": "Roman-Urdu sentiment",
      "decisions": 100,
      "scores": {
        "laya (English)": 0.36,
        "laya-multilingual": 0.41,
        "laya (routed)": 0.36,
        "jev (TypeSafe)": 0.59,
        "jev-urdu": 0.43,
        "majority class": 0.43
      }
    },
    {
      "prompt": "English",
      "benchmark": "XNLI-ur",
      "decisions": 100,
      "scores": {
        "laya (English)": 0.4,
        "laya-multilingual": 0.66,
        "laya (routed)": 0.66,
        "jev (TypeSafe)": 0.71,
        "jev-urdu": 0.69,
        "majority class": 0.35
      }
    },
    {
      "prompt": "English",
      "benchmark": "MASSIVE ur-PK",
      "decisions": 100,
      "scores": {
        "laya (English)": 0.08,
        "laya-multilingual": 0.37,
        "laya (routed)": 0.37,
        "jev (TypeSafe)": 0.79,
        "jev-urdu": 0.35,
        "majority class": 0.14
      }
    },
    {
      "prompt": "English",
      "benchmark": "Roman-Urdu sentiment",
      "decisions": 100,
      "scores": {
        "laya (English)": 0.56,
        "laya-multilingual": 0.56,
        "laya (routed)": 0.54,
        "jev (TypeSafe)": 0.67,
        "jev-urdu": 0.58,
        "majority class": 0.56
      }
    }
  ],
  "headToHeadDeltas": [
    {
      "benchmark": "Laya-Urdu test",
      "prompt": "Urdu",
      "delta": 0.177,
      "low": 0.129,
      "high": 0.228,
      "significant": true
    },
    {
      "benchmark": "XNLI-ur",
      "prompt": "Urdu",
      "delta": -0.08,
      "low": -0.19,
      "high": 0.03,
      "significant": false
    },
    {
      "benchmark": "MASSIVE ur-PK",
      "prompt": "Urdu",
      "delta": -0.48,
      "low": -0.59,
      "high": -0.38,
      "significant": true
    },
    {
      "benchmark": "Roman-Urdu sentiment",
      "prompt": "Urdu",
      "delta": -0.16,
      "low": -0.3,
      "high": -0.03,
      "significant": true
    },
    {
      "benchmark": "XNLI-ur",
      "prompt": "English",
      "delta": -0.02,
      "low": -0.11,
      "high": 0.08,
      "significant": false
    },
    {
      "benchmark": "MASSIVE ur-PK",
      "prompt": "English",
      "delta": -0.44,
      "low": -0.55,
      "high": -0.34,
      "significant": true
    },
    {
      "benchmark": "Roman-Urdu sentiment",
      "prompt": "English",
      "delta": -0.09,
      "low": -0.21,
      "high": 0.03,
      "significant": false
    }
  ],
  "throughput": [
    {
      "model": "laya (English)",
      "decisions": 22775,
      "seconds": 265.469,
      "decisionsPerSecond": 85.792
    },
    {
      "model": "laya-multilingual",
      "decisions": 22775,
      "seconds": 81.386,
      "decisionsPerSecond": 279.84
    },
    {
      "model": "jev-urdu",
      "decisions": 22775,
      "seconds": 81.551,
      "decisionsPerSecond": 279.272
    }
  ],
  "notes": [
    "Local accuracy summaries verified against 91,100 exported per-decision probability records.",
    "Full-suite results and the 100-item-per-suite API comparison use different sample sizes and must be kept separate.",
    "Internal workflow tests are template-based and do not establish real-world accuracy.",
    "English refers to question/option language; passage text remains Urdu or Roman Urdu.",
    "95% intervals use 2,000 paired bootstrap resamples of decisions.",
    "Published throughput is batch inference, not single-request latency or API throughput.",
    "Head-to-head scores and timing are transcribed from the saved report, which rounds them to three decimals."
  ],
  "encoderModel": "jhu-clsp/mmBERT-base",
  "startingCheckpoint": {
    "repository": "convaiinnovations/laya",
    "subfolder": "multilingual"
  },
  "modelSizes": [
    {
      "model": "jev-urdu",
      "parameterLabel": "321.9M",
      "weightLabel": "643.8 MB",
      "weightBytes": 643835514,
      "sourceUrl": "https://huggingface.co/muhammadnoman76/jev-urdu/tree/main"
    },
    {
      "model": "laya-multilingual",
      "parameterLabel": "321.9M",
      "weightLabel": "643.8 MB",
      "weightBytes": 643835514,
      "sourceUrl": "https://huggingface.co/convaiinnovations/laya/tree/main/multilingual"
    },
    {
      "model": "laya (English)",
      "parameterLabel": "421.3M",
      "weightLabel": "842.6 MB",
      "weightBytes": 842609210,
      "sourceUrl": "https://huggingface.co/convaiinnovations/laya/tree/main"
    },
    {
      "model": "laya (routed)",
      "parameterLabel": "321.9M or 421.3M per request",
      "weightLabel": "1.49 GB for both checkpoints",
      "weightBytes": 1486444724,
      "sourceUrl": "https://huggingface.co/convaiinnovations/laya"
    },
    {
      "model": "jev (TypeSafe)",
      "parameterLabel": "Not publicly disclosed",
      "weightLabel": "Not available for download",
      "weightBytes": null,
      "sourceUrl": "https://docs.typesafe.ai/models"
    }
  ]
}