The 69 build tasks of the harm check.
Most are features re-implemented from their pull-request description inside the repository's parent commit, with the commit's own tests held out; the rest are greenfield projects built from a written specification. Scores are the mean share of hidden tests passed, stock Claude Code and with the setup.
| Task | Source | Stack | Hidden tests | Stock | Setup |
|---|---|---|---|---|---|
Lightning-AI__pytorch-lightning-eea74433ce | Lightning-AI/pytorch-lightning | py | 16 | 0.75 | 0.69 |
PyCQA__pyflakes-d539458de5 | PyCQA/pyflakes | py | 10 | 1.00 | 1.00 |
aio-libs__yarl-09e500de4b | aio-libs/yarl | py | 12 | 1.00 | 1.00 |
alecthomas__kong-4e565e8d77 | alecthomas/kong | go | 1 | 1.00 | 1.00 |
gdamore__tcell-d57374f097 | gdamore/tcell | go | 2 | 1.00 | 1.00 |
go-resty__resty-29010be3b2 | go-resty/resty | go | 1 | 1.00 | 1.00 |
golang-jwt__jwt-73c870b18e | golang-jwt/jwt | go | 1 | 1.00 | 1.00 |
h3js__rou3-01bb393e2b | h3js/rou3 | ts | 4 | 1.00 | 1.00 |
h3js__rou3-79480e642f | h3js/rou3 | ts | 1 | 1.00 | 1.00 |
h3js__rou3-835332aca0 | h3js/rou3 | ts | 12 | 1.00 | 1.00 |
h3js__srvx-555a7932ae | h3js/srvx | ts | 3 | 1.00 | 1.00 |
h3js__srvx-6b3b2910ae | h3js/srvx | ts | 46 | 0.89 | 0.89 |
h3js__srvx-e0934021d4 | h3js/srvx | ts | 18 | 1.00 | 1.00 |
hgrecco__pint-bbc6b4c600 | hgrecco/pint | py | 1 | 0.00 | 0.00 |
joke2k__faker-4ca9116ec2 | joke2k/faker | py | 2 | 1.00 | 1.00 |
junegunn__fzf-f779e6a4df | junegunn/fzf | go | 2 | 1.00 | 1.00 |
labstack__echo-abbb92edbb | labstack/echo | go | 2 | 1.00 | 1.00 |
mark3labs__mcp-go-e9a7badcce | mark3labs/mcp-go | go | 7 | 1.00 | 0.50 |
ml-explore__mlx-lm-e9308d7b08 | ml-explore/mlx-lm | py | 5 | 1.00 | 1.00 |
modelcontextprotocol__go-sdk-3f43bcf012 | modelcontextprotocol/go-sdk | go | 1 | 1.00 | 1.00 |
mswjs__interceptors-4c01ff252e | mswjs/interceptors | ts | 6 | 1.00 | 1.00 |
mswjs__interceptors-7f1b37ca80 | mswjs/interceptors | ts | 6 | 0.67 | 0.67 |
openai__openai-go-7e4e1b1912 | openai/openai-go | go | 2 | 0.50 | 0.50 |
openai__openai-go-a29869f512 | openai/openai-go | go | 10 | 0.60 | 0.60 |
pypa__packaging-23669b1b69 | pypa/packaging | py | 2 | 0.50 | 0.50 |
pypa__packaging-58c6cd700b | pypa/packaging | py | 54 | 0.89 | 0.83 |
python-attrs__attrs-5aa76a4450 | python-attrs/attrs | py | 3 | 1.00 | 1.00 |
slice-go-cli-request-latency-report | greenfield project from a written spec | go | 13 | 1.00 | 1.00 |
slice-go-cli-sample-crate-planner | greenfield project from a written spec | go | 40 | 1.00 | 1.00 |
slice-go-web-gallery-cue-catalog | greenfield project from a written spec | go | 35 | 1.00 | 1.00 |
slice-go-web-signed-event-gateway | greenfield project from a written spec | go | 12 | 1.00 | 1.00 |
slice-go-worker-digest-packet-worker | greenfield project from a written spec | go | 36 | 1.00 | 1.00 |
slice-go-worker-revision-rebuild-worker | greenfield project from a written spec | go | 11 | 1.00 | 1.00 |
slice-py-cli-inventory-journal-fold | greenfield project from a written spec | py | 12 | 1.00 | 1.00 |
slice-py-cli-prefix-policy-forge | greenfield project from a written spec | py | 40 | 0.95 | 0.95 |
slice-py-web-buffered-dock-board | greenfield project from a written spec | py | 40 | 1.00 | 1.00 |
slice-py-web-gallery-caption-press | greenfield project from a written spec | py | 40 | 0.97 | 0.97 |
slice-py-web-harbor-berth-ledger | greenfield project from a written spec | py | 38 | 1.00 | 1.00 |
slice-py-web-incident-timeline-pages | greenfield project from a written spec | py | 12 | 1.00 | 1.00 |
slice-py-worker-checkpointed-maintenance-worker | greenfield project from a written spec | py | 12 | 1.00 | 0.92 |
slice-py-worker-porchlight-courier | greenfield project from a written spec | py | 49 | 1.00 | 1.00 |
slice-ts-cli-asset-revision-scout | greenfield project from a written spec | ts | 38 | 1.00 | 0.95 |
slice-ts-cli-catalog-placeholder-audit | greenfield project from a written spec | ts | 12 | 1.00 | 1.00 |
slice-ts-cli-deploy-wave-planner | greenfield project from a written spec | ts | 12 | 1.00 | 1.00 |
slice-ts-cli-env-overlay-plan | greenfield project from a written spec | ts | 12 | 1.00 | 1.00 |
slice-ts-cli-redirect-chain-auditor | greenfield project from a written spec | ts | 10 | 1.00 | 1.00 |
slice-ts-cli-support-log-scrubber | greenfield project from a written spec | ts | 12 | 1.00 | 1.00 |
slice-ts-web-apiary-inspection-api | greenfield project from a written spec | ts | 50 | 1.00 | 1.00 |
slice-ts-web-artifact-manifest-registry | greenfield project from a written spec | ts | 13 | 1.00 | 1.00 |
slice-ts-web-atomic-lane-layout | greenfield project from a written spec | ts | 12 | 1.00 | 1.00 |
slice-ts-web-durable-dispatch-desk | greenfield project from a written spec | ts | 40 | 1.00 | 1.00 |
slice-ts-web-leased-review-queue | greenfield project from a written spec | ts | 12 | 1.00 | 1.00 |
slice-ts-web-metered-usage-ledger | greenfield project from a written spec | ts | 42 | 1.00 | 1.00 |
slice-ts-web-restoration-queue-api | greenfield project from a written spec | ts | 35 | 1.00 | 1.00 |
slice-ts-web-transient-note-vault | greenfield project from a written spec | ts | 41 | 1.00 | 1.00 |
slice-ts-web-weekly-availability-api | greenfield project from a written spec | ts | 12 | 1.00 | 1.00 |
slice-ts-worker-attachment-scan-worker | greenfield project from a written spec | ts | 12 | 1.00 | 1.00 |
slice-ts-worker-caption-proof-conveyor | greenfield project from a written spec | ts | 40 | 1.00 | 1.00 |
slice-ts-worker-mirror-outbox-worker | greenfield project from a written spec | ts | 12 | 1.00 | 1.00 |
slice-ts-worker-receipt-gated-workflows | greenfield project from a written spec | ts | 12 | 1.00 | 1.00 |
slice-ts-worker-tenant-export-dispatcher | greenfield project from a written spec | ts | 13 | 1.00 | 1.00 |
stanfordnlp__dspy-33aaa19e03 | stanfordnlp/dspy | py | 8 | 0.88 | 0.81 |
stanfordnlp__dspy-d58c3007eb | stanfordnlp/dspy | py | 12 | 0.33 | 0.33 |
stanfordnlp__dspy-e8ec009c6b | stanfordnlp/dspy | py | 31 | 0.24 | 0.34 |
tobymao__sqlglot-da9a9c9664 | tobymao/sqlglot | py | 18 | 0.67 | 0.64 |
tobymao__sqlglot-e3ed74860f | tobymao/sqlglot | py | 2 | 0.50 | 0.50 |
tox-dev__filelock-126cab7456 | tox-dev/filelock | py | 31 | 0.74 | 0.74 |
tox-dev__platformdirs-dfaeabf921 | tox-dev/platformdirs | py | 7 | 0.86 | 0.86 |
yuin__goldmark-ee62f0eaf9 | yuin/goldmark | go | 2 | 0.00 | 0.50 |