{
 "meta": {
  "id": "corpora",
  "title": "Every corpus: sizes, splits and licence constraints",
  "sources": [
   "deterministic.html#1, #4–#5",
   "methodology.html#1",
   "cross-tool.html (records scored)",
   "reproduce.html (prose)",
   {
    "repo": "Vineethsain/skill-scanner-vs-skillspector",
    "path": "results/large-scale.json",
    "sha256": "e48f9cb342c05d2d13286c6b91c165411994d6febd71e14d1fe54c6f339d32b5"
   }
  ],
  "extracted_at": "2026-10-16T14:34:41+00:00",
  "notes": [
   "corpora: one entry per population with counts and the figure its licence permits.",
   "records_scored_on_both: per cross-tool arm, how many records both tools returned usable results for (from skillspector.json).",
   "remeasured_baseline / not_remeasured: deterministic.html#4 and #5."
  ],
  "page_sha256": {
   "deterministic.html": "752550a10cd6b5f5a0c465f1de4da1168521cefc519bedba7cd5c006f67e4cc4",
   "cross-tool.html": "e7f8433087a55e382dc51895d350412a45ce19124d12beda376a47a1d55509aa",
   "methodology.html": "a7a87af52e9ed8b91f308bf430c8247a235812bb6673c06ac189580cca8d2d3e",
   "reproduce.html": "95e1be0a878f1f106476790d86c38e29a1e0ec5a836c0f8434f9fb6ff5c81c99"
  }
 },
 "data": {
  "corpora": [
   {
    "id": "msb-source-disjoint",
    "name": "MaliciousSkillBench, source-disjoint split (frozen test)",
    "records": 1384,
    "malicious": 839,
    "benign": 545,
    "role": "held-out test split; used to design nothing",
    "licence": "ProtectSkills/MaliciousSkillBench terms: no threshold selection on test members; ATR pack excluded from source-disjoint claims",
    "population_sha256": "e77564f010fe55ee368af237fabc89b94ca308bd0b3a8c49f052716609f89399",
    "population_sha256_source": "deterministic.html prose (identifier, not a measurement)",
    "sources": [
     "deterministic.html#1",
     "results/large-scale.json full.labelled_ab"
    ]
   },
   {
    "id": "msb-trainval",
    "name": "MaliciousSkillBench train/validation (development split)",
    "records": 6594,
    "malicious": 5256,
    "benign": 1338,
    "role": "rules were designed on it; judge dev set is a 1,938-record subset (600 malicious, 1,338 benign)",
    "judge_dev": {
     "records": 1938,
     "malicious": 600,
     "benign": 1338
    },
    "sources": [
     "results/large-scale.json full.labelled_ab, jev_screen.records",
     "large-scale.html#11 and prose"
    ]
   },
   {
    "id": "msb-balanced-800",
    "name": "MaliciousSkillBench, label-balanced subset",
    "records": 800,
    "malicious": 400,
    "benign": 400,
    "role": "home ground: overlaps the test split (137 records); rule packs were tuned on MSB during development",
    "sources": [
     "cascade.html",
     "results/large-scale.json full.labelled_ab"
    ]
   },
   {
    "id": "openskillrisk-corpus",
    "name": "OpenSkillRisk",
    "records": 374,
    "contextually_risky": 214,
    "obviously_malicious": 160,
    "licence": "positive-risk only: detection rate permitted, no false-positive-rate claim",
    "sources": [
     "deterministic.html#3",
     "experiments.html#14"
    ]
   },
   {
    "id": "harmfulskillbench-corpus",
    "name": "HarmfulSkillBench",
    "records": 200,
    "licence": "flag rates only; terms forbid F1 and false-positive-rate claims",
    "note": "Cisco-side cross-tool figures predate a fix that moved a label-bearing _meta.json out of scan scope (label-contaminated).",
    "sources": [
     "deterministic.html#3",
     "cross-tool.html (run notes)"
    ]
   },
   {
    "id": "mcp-tool-poisoning",
    "name": "MCP tool-poisoning",
    "records": 50,
    "malicious": 25,
    "benign": 25,
    "sources": [
     "results/large-scale.json full.labelled_ab"
    ]
   },
   {
    "id": "gitskills-full",
    "name": "gitskills (mvaccargiu/gitskills), every usable skill",
    "records": 1876662,
    "rows": 1876769,
    "judge_genuine": 1876206,
    "errors_deterministic": 107,
    "role": "unlabelled: flag rates bound the false-positive rate from above",
    "sources": [
     "results/large-scale.json full"
    ]
   },
   {
    "id": "gitskills-sample",
    "name": "gitskills sample (labelled_ab)",
    "records": 199998,
    "role": "unlabelled",
    "sources": [
     "results/large-scale.json full.labelled_ab"
    ]
   },
   {
    "id": "real-world-skills",
    "name": "Real published skills (abersbail/ai-skill-md-dataset-500, MIT; FayeZC/SkillMD-138K, CC-BY-4.0)",
    "records": 12498,
    "sampled": 12500,
    "role": "unlabelled; SkillSpector run on 4,379 of them",
    "sources": [
     "improvements.html#8",
     "results/large-scale.json full.labelled_ab"
    ]
   },
   {
    "id": "real-2000",
    "name": "2,000 real skills sampled uniformly from gitskills",
    "records": 2000,
    "role": "judge and screen flag rates",
    "sources": [
     "results/large-scale.json jev_screen.records, judge_prompt"
    ]
   },
   {
    "id": "notinject",
    "name": "NotInject",
    "records": 339,
    "role": "benign text cases (re-measured baseline)",
    "sources": [
     "deterministic.html#4"
    ]
   },
   {
    "id": "injecagent",
    "name": "InjecAgent",
    "records": 1054,
    "role": "canonical signals (re-measured baseline)",
    "sources": [
     "deterministic.html#4"
    ]
   },
   {
    "id": "in-page-prompt-injection",
    "name": "In-Page Prompt Injection",
    "records": 1101,
    "role": "canonical groups (re-measured baseline)",
    "sources": [
     "deterministic.html#4"
    ]
   },
   {
    "id": "bundled-skills",
    "name": "Bundled skills snapshot",
    "records": 111,
    "role": "not re-measured: stale lock",
    "sources": [
     "deterministic.html#5"
    ]
   },
   {
    "id": "datadog-malicious",
    "name": "DataDog malicious packages",
    "records": 5,
    "role": "not re-measured: quarantine-only",
    "sources": [
     "deterministic.html#5"
    ]
   },
   {
    "id": "malicious-agent-skills-bench",
    "name": "MaliciousAgentSkillsBench",
    "records": 98380,
    "role": "not re-measured: metadata only",
    "sources": [
     "deterministic.html#5"
    ]
   }
  ],
  "records_scored_on_both": [
   {
    "arm": "gemma-4-26b",
    "corpus": "msb-source-disjoint",
    "records_scored_on_both": 1367,
    "composition": {
     "benign": 528,
     "malicious": 839
    },
    "source": "cross-tool.html (section intro line)"
   },
   {
    "arm": "gemma-4-26b",
    "corpus": "msb-balanced-800",
    "records_scored_on_both": 793,
    "composition": {
     "benign": 397,
     "malicious": 396
    },
    "source": "cross-tool.html (section intro line)"
   },
   {
    "arm": "gemma-4-26b",
    "corpus": "openskillrisk-corpus",
    "records_scored_on_both": 343,
    "composition": {
     "contextually_risky": 188,
     "obviously_malicious": 155
    },
    "source": "cross-tool.html (section intro line)"
   },
   {
    "arm": "gemma-4-26b",
    "corpus": "harmfulskillbench-corpus",
    "records_scored_on_both": 196,
    "composition": {
     "unlabelled": 196
    },
    "source": "cross-tool.html (section intro line)"
   },
   {
    "arm": "claude-haiku-4.5",
    "corpus": "msb-source-disjoint",
    "records_scored_on_both": 1322,
    "composition": {
     "benign": 518,
     "malicious": 804
    },
    "source": "cross-tool.html (section intro line)"
   },
   {
    "arm": "claude-haiku-4.5",
    "corpus": "msb-balanced-800",
    "records_scored_on_both": 766,
    "composition": {
     "benign": 388,
     "malicious": 378
    },
    "source": "cross-tool.html (section intro line)"
   },
   {
    "arm": "static",
    "corpus": "msb-source-disjoint",
    "records_scored_on_both": 1368,
    "composition": {
     "benign": 529,
     "malicious": 839
    },
    "source": "cross-tool.html (section intro line)"
   },
   {
    "arm": "static",
    "corpus": "msb-balanced-800",
    "records_scored_on_both": 794,
    "composition": {
     "benign": 397,
     "malicious": 397
    },
    "source": "cross-tool.html (section intro line)"
   },
   {
    "arm": "static",
    "corpus": "openskillrisk-corpus",
    "records_scored_on_both": 335,
    "composition": {
     "contextually_risky": 182,
     "obviously_malicious": 153
    },
    "source": "cross-tool.html (section intro line)"
   },
   {
    "arm": "static",
    "corpus": "harmfulskillbench-corpus",
    "records_scored_on_both": 197,
    "composition": {
     "unlabelled": 197
    },
    "source": "cross-tool.html (section intro line)"
   }
  ],
  "remeasured_baseline": [
   {
    "corpus": "MaliciousSkillBench source-disjoint",
    "population": {
     "n": 1384,
     "unit": "packages"
    },
    "previously_published": {
     "raw": "F1 13.74%",
     "type": "labelled",
     "value": [
      0.1374
     ],
     "col": "Previously published"
    },
    "re_measured": {
     "raw": "F1 13.742%",
     "type": "labelled",
     "value": [
      0.13742
     ],
     "col": "Re-measured"
    },
    "agrees": true
   },
   {
    "corpus": "MaliciousSkillBench source-disjoint",
    "population": {
     "n": 1384,
     "unit": "packages"
    },
    "previously_published": {
     "raw": "precision 60.75%",
     "type": "labelled",
     "value": [
      0.6075
     ],
     "col": "Previously published"
    },
    "re_measured": {
     "raw": "60.748%",
     "type": "percent",
     "value": 0.60748,
     "col": "Re-measured"
    },
    "agrees": true
   },
   {
    "corpus": "MaliciousSkillBench source-disjoint",
    "population": {
     "n": 1384,
     "unit": "packages"
    },
    "previously_published": {
     "raw": "recall 7.75%",
     "type": "labelled",
     "value": [
      0.0775
     ],
     "col": "Previously published"
    },
    "re_measured": {
     "raw": "7.747%",
     "type": "percent",
     "value": 0.07747,
     "col": "Re-measured"
    },
    "agrees": true
   },
   {
    "corpus": "MaliciousSkillBench source-disjoint",
    "population": {
     "n": 1384,
     "unit": "packages"
    },
    "previously_published": {
     "raw": "FPR 7.71%",
     "type": "labelled",
     "value": [
      0.0771
     ],
     "col": "Previously published"
    },
    "re_measured": {
     "raw": "7.706%",
     "type": "percent",
     "value": 0.07706,
     "col": "Re-measured"
    },
    "agrees": true
   },
   {
    "corpus": "NotInject",
    "population": {
     "n": 339,
     "unit": "benign text cases"
    },
    "previously_published": {
     "raw": "0.00% flag rate at MEDIUM+",
     "type": "labelled",
     "value": [
      0.0
     ],
     "col": "Previously published"
    },
    "re_measured": {
     "raw": "0.00%",
     "type": "percent",
     "value": 0.0,
     "col": "Re-measured"
    },
    "agrees": true
   },
   {
    "corpus": "InjecAgent",
    "population": {
     "n": 1054,
     "unit": "canonical signals"
    },
    "previously_published": {
     "raw": "100.00% signal recall",
     "type": "labelled",
     "value": [
      1.0
     ],
     "col": "Previously published"
    },
    "re_measured": {
     "raw": "100.00%",
     "type": "percent",
     "value": 1.0,
     "col": "Re-measured"
    },
    "agrees": true
   },
   {
    "corpus": "In-Page Prompt Injection",
    "population": {
     "n": 1101,
     "unit": "canonical groups"
    },
    "previously_published": {
     "raw": "99.36% signal recall",
     "type": "labelled",
     "value": [
      0.9936
     ],
     "col": "Previously published"
    },
    "re_measured": {
     "raw": "99.36%",
     "type": "percent",
     "value": 0.9936,
     "col": "Re-measured"
    },
    "agrees": true
   },
   {
    "corpus": "HarmfulSkillBench",
    "population": {
     "n": 200,
     "unit": "positive-risk skills"
    },
    "previously_published": {
     "raw": "3.50% MEDIUM+",
     "type": "labelled",
     "value": [
      0.035
     ],
     "col": "Previously published"
    },
    "re_measured": {
     "raw": "3.5% block rate",
     "type": "labelled",
     "value": [
      0.035
     ],
     "col": "Re-measured"
    },
    "agrees": true
   },
   {
    "corpus": "OpenSkillRisk",
    "population": {
     "n": 374,
     "unit": "positive-risk skills"
    },
    "previously_published": {
     "raw": "28.90% HIGH+, 35.74% MEDIUM+",
     "type": "labelled",
     "value": [
      0.289,
      0.3574
     ],
     "col": "Previously published"
    },
    "re_measured": {
     "raw": "30.0% and 32.7% detection",
     "type": "labelled",
     "value": [
      0.3,
      0.327
     ],
     "col": "Re-measured"
    },
    "agrees": true
   }
  ],
  "not_remeasured": [
   {
    "corpus": "Bundled skills snapshot",
    "population": {
     "n": 111,
     "unit": "installed skills"
    },
    "reason": "the lock is stale against the currently installed applications, and refreshing it requires human review; forcing a refresh would change the corpus identity and break comparability"
   },
   {
    "corpus": "DataDog malicious packages",
    "population": {
     "n": 5,
     "unit": "selected positives"
    },
    "reason": "quarantine-only acquisition policy, and too small to move any figure"
   },
   {
    "corpus": "MaliciousAgentSkillsBench",
    "population": {
     "n": 98380,
     "unit": "rows"
    },
    "reason": "metadata only, with no skill content to scan"
   }
  ],
  "shape": [
   {
    "population": "MaliciousSkillBench rows",
    "median_files": "1 (single text field)",
    "median_scanner_relevant_bytes": 2936,
    "fits_32k_window": "essentially all"
   },
   {
    "population": "ClawHub rows",
    "median_files": "2 (skill_md + bundle)",
    "median_scanner_relevant_bytes": 24173,
    "fits_32k_window": "most"
   },
   {
    "population": "Golden fixtures",
    "median_files": "3",
    "median_scanner_relevant_bytes": 1797,
    "fits_32k_window": "100%"
   },
   {
    "population": "Real-world skills",
    "median_files": "80",
    "median_scanner_relevant_bytes": 576745,
    "fits_32k_window": "16%"
   }
  ],
  "packing_ladder": {
   "rung_A": "sends the whole record when it fits (essentially every benchmark row)",
   "rung_B": "fills the budget by priority, SKILL.md first",
   "rung_C": "one request per candidate finding",
   "source": "methodology.html prose (shares listed as gaps)"
  }
 }
}