{ "schema_version": "evalarc.audit.v2", "valid": true, "reference_passed": true, "reference": { "schema_version": "evalarc.evaluation.v2", "created_at": "2026-09-14T01:29:51.125110+00:00", "evalarc_version": "0.2.0", "task": { "id": "durable-kv", "version": "0.1.0", "domain": "coding", "split": "public-development" }, "candidate_sha256": "4240da0ab584ebbda01a737b07bb351ee465fe494595b96f67fd3371daf0f3f2", "grader_sha256": "52caa3136434569f96097860bc9bf85eb7ada5f7f9890c0540db81afb441871d", "cases_sha256": "842a16a1e65700cde92ee5f5aaaa10d494a9fc9d71bd2d99fb68147b19aad63a", "runtime": { "backend": "docker", "image": "python:3.12-slim", "image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea", "command": [ "{python}", "-I", "-B", "main.py", "{state}/store.db" ], "response_timeout_seconds": 10.0, "session_output_limit_bytes": 1048576, "python": "3.12.3", "platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39" }, "seeds": [ 17 ], "valid": true, "status": "passed", "score": 1.0, "resolved": true, "dimensions": { "basic": { "passed": 3, "total": 3, "assessed": 3, "weight": 0.25, "score": 1.0 }, "validation": { "passed": 2, "total": 2, "assessed": 2, "weight": 0.15, "score": 1.0 }, "transactions": { "passed": 2, "total": 2, "assessed": 2, "weight": 0.2, "score": 1.0 }, "compare_swap": { "passed": 2, "total": 2, "assessed": 2, "weight": 0.15, "score": 1.0 }, "persistence": { "passed": 3, "total": 3, "assessed": 3, "weight": 0.15, "score": 1.0 }, "crash_recovery": { "passed": 3, "total": 3, "assessed": 3, "weight": 0.1, "score": 1.0 } }, "cases": [ { "seed": 17, "case_id": "read-write-delete", "dimension": "basic", "checks": { "basic": true }, "passed": true, "status": "passed", "checks_completed": 6, "duration_seconds": 0.926471, "transcript_sha256": "c809eed26dc56583f9385d7d4adca2d981d06a672dc1ccbc0ffc52aa228b86f7", "error": null }, { "seed": 17, "case_id": "json-types-and-overwrite", "dimension": "basic", "checks": { "basic": true }, "passed": true, "status": "passed", "checks_completed": 16, "duration_seconds": 0.992022, "transcript_sha256": "83f6377f4b57ac659e8162136107ca644158ee089c944892aa9817085e74bd60", "error": null }, { "seed": 17, "case_id": "reject-and-continue", "dimension": "validation", "checks": { "validation": true }, "passed": true, "status": "passed", "checks_completed": 10, "duration_seconds": 0.802672, "transcript_sha256": "d4e0c26837f54e0af94d4909e3cab21d86dedca8c8842d0daae9e7471198c2a0", "error": null }, { "seed": 17, "case_id": "empty-and-unicode-keys", "dimension": "validation", "checks": { "validation": true }, "passed": true, "status": "passed", "checks_completed": 4, "duration_seconds": 0.668748, "transcript_sha256": "e8abe374fc308ce0744e96d2920495b96bda3b026cec9b86fa055023e1e4e620", "error": null }, { "seed": 17, "case_id": "commit-batch", "dimension": "transactions", "checks": { "transactions": true }, "passed": true, "status": "passed", "checks_completed": 4, "duration_seconds": 0.654542, "transcript_sha256": "ac21f2f6bcc47e3c807fa4ed72300af0f5ef39809ba17f93423e37a0e177e718", "error": null }, { "seed": 17, "case_id": "rollback-batch", "dimension": "transactions", "checks": { "transactions": true }, "passed": true, "status": "passed", "checks_completed": 7, "duration_seconds": 0.69832, "transcript_sha256": "1786c093ba29dcad562531a44cdbe5fddbed9d3f4632c1c6ddfe736baa240f35", "error": null }, { "seed": 17, "case_id": "cas-match-and-mismatch", "dimension": "compare_swap", "checks": { "compare_swap": true }, "passed": true, "status": "passed", "checks_completed": 7, "duration_seconds": 0.662942, "transcript_sha256": "ecd6148f6f387b4ef5351b877afd4cf27ea2a04c4253a34220fd78de5442d89f", "error": null }, { "seed": 17, "case_id": "cas-type-sensitivity", "dimension": "compare_swap", "checks": { "compare_swap": true }, "passed": true, "status": "passed", "checks_completed": 5, "duration_seconds": 0.749806, "transcript_sha256": "459a5b74588217d8abcf8e138eb559364a1d01d1f23e8c43ca0127ba28240258", "error": null }, { "seed": 17, "case_id": "restart-preserves-acknowledged-write", "dimension": "persistence", "checks": { "persistence": true }, "passed": true, "status": "passed", "checks_completed": 4, "duration_seconds": 2.199417, "transcript_sha256": "c97b82c67f26a1aa2fdc8ac57f50690dbbe13554d094059ffe6690a592df3a55", "error": null }, { "seed": 17, "case_id": "restart-preserves-transaction", "dimension": "persistence", "checks": { "persistence": true }, "passed": true, "status": "passed", "checks_completed": 3, "duration_seconds": 1.166844, "transcript_sha256": "2d1f704df309d49f8e6e89e674581af2cf81aa22fdde633058e2f2d7b8c8a430", "error": null }, { "seed": 17, "case_id": "sigkill-after-acknowledgement", "dimension": "crash_recovery", "checks": { "crash_recovery": true }, "passed": true, "status": "passed", "checks_completed": 2, "duration_seconds": 1.224396, "transcript_sha256": "544c6be916371ed5a0e901394c5e293733ca84e0ad20079a952f9c4b2821a728", "error": null }, { "seed": 17, "case_id": "sigkill-after-atomic-batch", "dimension": "crash_recovery", "checks": { "crash_recovery": true }, "passed": true, "status": "passed", "checks_completed": 3, "duration_seconds": 1.151733, "transcript_sha256": "2d1f704df309d49f8e6e89e674581af2cf81aa22fdde633058e2f2d7b8c8a430", "error": null }, { "seed": 17, "case_id": "restart-preserves-cas", "dimension": "persistence", "checks": { "persistence": true }, "passed": true, "status": "passed", "checks_completed": 3, "duration_seconds": 1.160644, "transcript_sha256": "149c04fd1366e77603e631349740ae38bdff40aa69f6eb4c738979eb13bf3f49", "error": null }, { "seed": 17, "case_id": "sigkill-after-delete", "dimension": "crash_recovery", "checks": { "crash_recovery": true }, "passed": true, "status": "passed", "checks_completed": 3, "duration_seconds": 1.718475, "transcript_sha256": "b3dba7f28e6629bb22f49b0439e0f74580ca3263dd98e16061ac9a8f680b5df7", "error": null }, { "seed": 17, "case_id": "seeded-state-machine", "dimension": "basic", "checks": { "basic": true }, "passed": true, "status": "passed", "checks_completed": 52, "duration_seconds": 0.815656, "transcript_sha256": "2a7e5ad4bedce67faf4d53f289c3a1a7304345a0167b287418fdf7dd411030f8", "error": null } ], "evaluation_seconds": 15.605099, "agent_cost_usd": null, "agent_tokens": null }, "mutants": [ { "name": "ack-without-work", "target_dimension": "basic", "killed": true, "valid": true, "score": 0.0, "failing_cases": [ "json-types-and-overwrite", "read-write-delete", "seeded-state-machine" ], "candidate_sha256": "e80e9d07c1b2a54e4e68a717c3ede2e71dbc871f7bc9c0174cffd9d0ef466f39", "evaluation": { "schema_version": "evalarc.evaluation.v2", "created_at": "2026-09-14T01:30:02.514728+00:00", "evalarc_version": "0.2.0", "task": { "id": "durable-kv", "version": "0.1.0", "domain": "coding", "split": "public-development" }, "candidate_sha256": "e80e9d07c1b2a54e4e68a717c3ede2e71dbc871f7bc9c0174cffd9d0ef466f39", "grader_sha256": "52caa3136434569f96097860bc9bf85eb7ada5f7f9890c0540db81afb441871d", "cases_sha256": "842a16a1e65700cde92ee5f5aaaa10d494a9fc9d71bd2d99fb68147b19aad63a", "runtime": { "backend": "docker", "image": "python:3.12-slim", "image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea", "command": [ "{python}", "-I", "-B", "main.py", "{state}/store.db" ], "response_timeout_seconds": 10.0, "session_output_limit_bytes": 1048576, "python": "3.12.3", "platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39" }, "seeds": [ 17 ], "valid": true, "status": "failed", "score": 0.0, "resolved": false, "dimensions": { "basic": { "passed": 0, "total": 3, "assessed": 3, "weight": 0.25, "score": 0.0 }, "validation": { "passed": 0, "total": 2, "assessed": 2, "weight": 0.15, "score": 0.0 }, "transactions": { "passed": 0, "total": 2, "assessed": 2, "weight": 0.2, "score": 0.0 }, "compare_swap": { "passed": 0, "total": 2, "assessed": 2, "weight": 0.15, "score": 0.0 }, "persistence": { "passed": 0, "total": 3, "assessed": 3, "weight": 0.15, "score": 0.0 }, "crash_recovery": { "passed": 0, "total": 3, "assessed": 3, "weight": 0.1, "score": 0.0 } }, "cases": [ { "seed": 17, "case_id": "read-write-delete", "dimension": "basic", "checks": { "basic": false }, "passed": false, "status": "failed", "checks_completed": 0, "duration_seconds": 0.608378, "transcript_sha256": "e9d1c1a4ce952f925f5a3d00a84348bf482007670ba13602f73d0129cc422a60", "error": "response mismatch at request 1; expected {\"found\": false, \"ok\": true}, got {\"ok\": true}" }, { "seed": 17, "case_id": "json-types-and-overwrite", "dimension": "basic", "checks": { "basic": false }, "passed": false, "status": "failed", "checks_completed": 1, "duration_seconds": 0.563549, "transcript_sha256": "ad684a2e4e62254b35236e279cda7ec03bfe6bf6a01e3c0b24a1810e6c8088d2", "error": "response mismatch at request 2; expected {\"found\": true, \"ok\": true, \"value\": null}, got {\"ok\": true}" }, { "seed": 17, "case_id": "reject-and-continue", "dimension": "validation", "checks": { "validation": false }, "passed": false, "status": "failed", "checks_completed": 1, "duration_seconds": 0.596346, "transcript_sha256": "932f3ec3f66ee636feddf02978271d1af20d352d113d51580182f4956132fcbc", "error": "response mismatch at request 2; expected {\"error\": \"invalid_request\", \"ok\": false}, got {\"ok\": true}" }, { "seed": 17, "case_id": "empty-and-unicode-keys", "dimension": "validation", "checks": { "validation": false }, "passed": false, "status": "failed", "checks_completed": 1, "duration_seconds": 0.576032, "transcript_sha256": "dd5c4b23bf9661f05363e4be2791dffa554d3bb7a6226d1be64e58adc97e7907", "error": "response mismatch at request 2; expected {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}, got {\"ok\": true}" }, { "seed": 17, "case_id": "commit-batch", "dimension": "transactions", "checks": { "transactions": false }, "passed": false, "status": "failed", "checks_completed": 1, "duration_seconds": 0.635267, "transcript_sha256": "660d89f2f1ccd8d228c1871197f23f08289d1c1fe244918053aa9aca3a9ce576", "error": "response mismatch at request 2; expected {\"found\": false, \"ok\": true}, got {\"ok\": true}" }, { "seed": 17, "case_id": "rollback-batch", "dimension": "transactions", "checks": { "transactions": false }, "passed": false, "status": "failed", "checks_completed": 1, "duration_seconds": 0.5925, "transcript_sha256": "8d05dbc72469fa9ab22e4943d0e9ed13a8733f443dc95a84a306daa02b2031b4", "error": "response mismatch at request 2; expected {\"error\": \"invalid_request\", \"ok\": false}, got {\"ok\": true}" }, { "seed": 17, "case_id": "cas-match-and-mismatch", "dimension": "compare_swap", "checks": { "compare_swap": false }, "passed": false, "status": "failed", "checks_completed": 1, "duration_seconds": 0.558764, "transcript_sha256": "92c53f111ed9d1c600b448aaf9e5a529f32d2f085433ad700cd369bd71c55185", "error": "response mismatch at request 2; expected {\"ok\": true, \"swapped\": false}, got {\"ok\": true}" }, { "seed": 17, "case_id": "cas-type-sensitivity", "dimension": "compare_swap", "checks": { "compare_swap": false }, "passed": false, "status": "failed", "checks_completed": 1, "duration_seconds": 0.689461, "transcript_sha256": "88cbf896bd041b5c3a80f99fc01f19d0b473a1a39cdb4f59d621a8abbcaa35e4", "error": "response mismatch at request 2; expected {\"ok\": true, \"swapped\": false}, got {\"ok\": true}" }, { "seed": 17, "case_id": "restart-preserves-acknowledged-write", "dimension": "persistence", "checks": { "persistence": false }, "passed": false, "status": "failed", "checks_completed": 1, "duration_seconds": 1.031052, "transcript_sha256": "d01e5e8ceee977002ba418a5bf134418cd85c9b36813ea2fadfa7908c141b718", "error": "response mismatch at request 2; expected {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}, got {\"ok\": true}" }, { "seed": 17, "case_id": "restart-preserves-transaction", "dimension": "persistence", "checks": { "persistence": false }, "passed": false, "status": "failed", "checks_completed": 1, "duration_seconds": 1.008206, "transcript_sha256": "5c70c6333e7c60969ecad930a4b839122a1582a89cdf257e323ffa6fbbf3036f", "error": "response mismatch at request 2; expected {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}, got {\"ok\": true}" }, { "seed": 17, "case_id": "sigkill-after-acknowledgement", "dimension": "crash_recovery", "checks": { "crash_recovery": false }, "passed": false, "status": "failed", "checks_completed": 1, "duration_seconds": 1.008229, "transcript_sha256": "d01e5e8ceee977002ba418a5bf134418cd85c9b36813ea2fadfa7908c141b718", "error": "response mismatch at request 2; expected {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}, got {\"ok\": true}" }, { "seed": 17, "case_id": "sigkill-after-atomic-batch", "dimension": "crash_recovery", "checks": { "crash_recovery": false }, "passed": false, "status": "failed", "checks_completed": 1, "duration_seconds": 1.010499, "transcript_sha256": "5c70c6333e7c60969ecad930a4b839122a1582a89cdf257e323ffa6fbbf3036f", "error": "response mismatch at request 2; expected {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}, got {\"ok\": true}" }, { "seed": 17, "case_id": "restart-preserves-cas", "dimension": "persistence", "checks": { "persistence": false }, "passed": false, "status": "failed", "checks_completed": 1, "duration_seconds": 0.506207, "transcript_sha256": "7ef0ed9705196a8514631dacec2c92eeb5e34cd9811d281f9816e348c50d1a17", "error": "response mismatch at request 2; expected {\"ok\": true, \"swapped\": true}, got {\"ok\": true}" }, { "seed": 17, "case_id": "sigkill-after-delete", "dimension": "crash_recovery", "checks": { "crash_recovery": false }, "passed": false, "status": "failed", "checks_completed": 2, "duration_seconds": 1.497816, "transcript_sha256": "165fee513b6ad33052636d17d8a1f492e3226d1b5f19b246f01f1c20289f5525", "error": "response mismatch at request 3; expected {\"found\": false, \"ok\": true}, got {\"ok\": true}" }, { "seed": 17, "case_id": "seeded-state-machine", "dimension": "basic", "checks": { "basic": false }, "passed": false, "status": "failed", "checks_completed": 2, "duration_seconds": 0.497166, "transcript_sha256": "ce6e264c70b36102aabe44145d2a7558766913a2725261b8f120efea98fa8659", "error": "response mismatch at request 3; expected {\"found\": false, \"ok\": true}, got {\"ok\": true}" } ], "evaluation_seconds": 11.386876, "agent_cost_usd": null, "agent_tokens": null } }, { "name": "memory-only", "target_dimension": "persistence", "killed": true, "valid": true, "score": 0.78333333, "failing_cases": [ "restart-preserves-acknowledged-write", "restart-preserves-cas", "restart-preserves-transaction" ], "candidate_sha256": "478fc026e785f7dcad2dd83744abbba6c482dc1f903e6ece8b0803dd335389e1", "evaluation": { "schema_version": "evalarc.evaluation.v2", "created_at": "2026-09-14T01:30:13.553761+00:00", "evalarc_version": "0.2.0", "task": { "id": "durable-kv", "version": "0.1.0", "domain": "coding", "split": "public-development" }, "candidate_sha256": "478fc026e785f7dcad2dd83744abbba6c482dc1f903e6ece8b0803dd335389e1", "grader_sha256": "52caa3136434569f96097860bc9bf85eb7ada5f7f9890c0540db81afb441871d", "cases_sha256": "842a16a1e65700cde92ee5f5aaaa10d494a9fc9d71bd2d99fb68147b19aad63a", "runtime": { "backend": "docker", "image": "python:3.12-slim", "image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea", "command": [ "{python}", "-I", "-B", "main.py", "{state}/store.db" ], "response_timeout_seconds": 10.0, "session_output_limit_bytes": 1048576, "python": "3.12.3", "platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39" }, "seeds": [ 17 ], "valid": true, "status": "failed", "score": 0.78333333, "resolved": false, "dimensions": { "basic": { "passed": 3, "total": 3, "assessed": 3, "weight": 0.25, "score": 1.0 }, "validation": { "passed": 2, "total": 2, "assessed": 2, "weight": 0.15, "score": 1.0 }, "transactions": { "passed": 2, "total": 2, "assessed": 2, "weight": 0.2, "score": 1.0 }, "compare_swap": { "passed": 2, "total": 2, "assessed": 2, "weight": 0.15, "score": 1.0 }, "persistence": { "passed": 0, "total": 3, "assessed": 3, "weight": 0.15, "score": 0.0 }, "crash_recovery": { "passed": 1, "total": 3, "assessed": 3, "weight": 0.1, "score": 0.3333333333333333 } }, "cases": [ { "seed": 17, "case_id": "read-write-delete", "dimension": "basic", "checks": { "basic": true }, "passed": true, "status": "passed", "checks_completed": 6, "duration_seconds": 0.510333, "transcript_sha256": "c809eed26dc56583f9385d7d4adca2d981d06a672dc1ccbc0ffc52aa228b86f7", "error": null }, { "seed": 17, "case_id": "json-types-and-overwrite", "dimension": "basic", "checks": { "basic": true }, "passed": true, "status": "passed", "checks_completed": 16, "duration_seconds": 0.512203, "transcript_sha256": "83f6377f4b57ac659e8162136107ca644158ee089c944892aa9817085e74bd60", "error": null }, { "seed": 17, "case_id": "reject-and-continue", "dimension": "validation", "checks": { "validation": true }, "passed": true, "status": "passed", "checks_completed": 10, "duration_seconds": 0.5002, "transcript_sha256": "d4e0c26837f54e0af94d4909e3cab21d86dedca8c8842d0daae9e7471198c2a0", "error": null }, { "seed": 17, "case_id": "empty-and-unicode-keys", "dimension": "validation", "checks": { "validation": true }, "passed": true, "status": "passed", "checks_completed": 4, "duration_seconds": 0.499981, "transcript_sha256": "e8abe374fc308ce0744e96d2920495b96bda3b026cec9b86fa055023e1e4e620", "error": null }, { "seed": 17, "case_id": "commit-batch", "dimension": "transactions", "checks": { "transactions": true }, "passed": true, "status": "passed", "checks_completed": 4, "duration_seconds": 0.487121, "transcript_sha256": "ac21f2f6bcc47e3c807fa4ed72300af0f5ef39809ba17f93423e37a0e177e718", "error": null }, { "seed": 17, "case_id": "rollback-batch", "dimension": "transactions", "checks": { "transactions": true }, "passed": true, "status": "passed", "checks_completed": 7, "duration_seconds": 0.492795, "transcript_sha256": "1786c093ba29dcad562531a44cdbe5fddbed9d3f4632c1c6ddfe736baa240f35", "error": null }, { "seed": 17, "case_id": "cas-match-and-mismatch", "dimension": "compare_swap", "checks": { "compare_swap": true }, "passed": true, "status": "passed", "checks_completed": 7, "duration_seconds": 0.491807, "transcript_sha256": "ecd6148f6f387b4ef5351b877afd4cf27ea2a04c4253a34220fd78de5442d89f", "error": null }, { "seed": 17, "case_id": "cas-type-sensitivity", "dimension": "compare_swap", "checks": { "compare_swap": true }, "passed": true, "status": "passed", "checks_completed": 5, "duration_seconds": 0.494448, "transcript_sha256": "459a5b74588217d8abcf8e138eb559364a1d01d1f23e8c43ca0127ba28240258", "error": null }, { "seed": 17, "case_id": "restart-preserves-acknowledged-write", "dimension": "persistence", "checks": { "persistence": false }, "passed": false, "status": "failed", "checks_completed": 1, "duration_seconds": 0.976095, "transcript_sha256": "f62cc3fb4db094e24ad23141f7cc50908d24647156c4d2ba2386a2487c078e80", "error": "response mismatch at request 2; expected {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}, got {\"found\": false, \"ok\": true}" }, { "seed": 17, "case_id": "restart-preserves-transaction", "dimension": "persistence", "checks": { "persistence": false }, "passed": false, "status": "failed", "checks_completed": 1, "duration_seconds": 0.996261, "transcript_sha256": "43cf3772fd4bd95486b1824b1d73c4e99859fb85088a7f62192d8325b4581b29", "error": "response mismatch at request 2; expected {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}, got {\"found\": false, \"ok\": true}" }, { "seed": 17, "case_id": "sigkill-after-acknowledgement", "dimension": "crash_recovery", "checks": { "crash_recovery": false }, "passed": false, "status": "failed", "checks_completed": 1, "duration_seconds": 1.054678, "transcript_sha256": "f62cc3fb4db094e24ad23141f7cc50908d24647156c4d2ba2386a2487c078e80", "error": "response mismatch at request 2; expected {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}, got {\"found\": false, \"ok\": true}" }, { "seed": 17, "case_id": "sigkill-after-atomic-batch", "dimension": "crash_recovery", "checks": { "crash_recovery": false }, "passed": false, "status": "failed", "checks_completed": 1, "duration_seconds": 1.013921, "transcript_sha256": "43cf3772fd4bd95486b1824b1d73c4e99859fb85088a7f62192d8325b4581b29", "error": "response mismatch at request 2; expected {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}, got {\"found\": false, \"ok\": true}" }, { "seed": 17, "case_id": "restart-preserves-cas", "dimension": "persistence", "checks": { "persistence": false }, "passed": false, "status": "failed", "checks_completed": 2, "duration_seconds": 0.997537, "transcript_sha256": "497082839111501c6aa4912312fe30686befcab05beb84ebbcb99be0d1e8aef1", "error": "response mismatch at request 3; expected {\"found\": true, \"ok\": true, \"value\": 321}, got {\"found\": false, \"ok\": true}" }, { "seed": 17, "case_id": "sigkill-after-delete", "dimension": "crash_recovery", "checks": { "crash_recovery": true }, "passed": true, "status": "passed", "checks_completed": 3, "duration_seconds": 1.494879, "transcript_sha256": "b3dba7f28e6629bb22f49b0439e0f74580ca3263dd98e16061ac9a8f680b5df7", "error": null }, { "seed": 17, "case_id": "seeded-state-machine", "dimension": "basic", "checks": { "basic": true }, "passed": true, "status": "passed", "checks_completed": 52, "duration_seconds": 0.509886, "transcript_sha256": "2a7e5ad4bedce67faf4d53f289c3a1a7304345a0167b287418fdf7dd411030f8", "error": null } ], "evaluation_seconds": 11.038733, "agent_cost_usd": null, "agent_tokens": null } }, { "name": "partial-batch", "target_dimension": "transactions", "killed": true, "valid": true, "score": 0.9, "failing_cases": [ "rollback-batch" ], "candidate_sha256": "833f852d0c053ed18f935d9daf03bb36a174f517c54fe75675f874b05036140a", "evaluation": { "schema_version": "evalarc.evaluation.v2", "created_at": "2026-09-14T01:30:25.787520+00:00", "evalarc_version": "0.2.0", "task": { "id": "durable-kv", "version": "0.1.0", "domain": "coding", "split": "public-development" }, "candidate_sha256": "833f852d0c053ed18f935d9daf03bb36a174f517c54fe75675f874b05036140a", "grader_sha256": "52caa3136434569f96097860bc9bf85eb7ada5f7f9890c0540db81afb441871d", "cases_sha256": "842a16a1e65700cde92ee5f5aaaa10d494a9fc9d71bd2d99fb68147b19aad63a", "runtime": { "backend": "docker", "image": "python:3.12-slim", "image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea", "command": [ "{python}", "-I", "-B", "main.py", "{state}/store.db" ], "response_timeout_seconds": 10.0, "session_output_limit_bytes": 1048576, "python": "3.12.3", "platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39" }, "seeds": [ 17 ], "valid": true, "status": "failed", "score": 0.9, "resolved": false, "dimensions": { "basic": { "passed": 3, "total": 3, "assessed": 3, "weight": 0.25, "score": 1.0 }, "validation": { "passed": 2, "total": 2, "assessed": 2, "weight": 0.15, "score": 1.0 }, "transactions": { "passed": 1, "total": 2, "assessed": 2, "weight": 0.2, "score": 0.5 }, "compare_swap": { "passed": 2, "total": 2, "assessed": 2, "weight": 0.15, "score": 1.0 }, "persistence": { "passed": 3, "total": 3, "assessed": 3, "weight": 0.15, "score": 1.0 }, "crash_recovery": { "passed": 3, "total": 3, "assessed": 3, "weight": 0.1, "score": 1.0 } }, "cases": [ { "seed": 17, "case_id": "read-write-delete", "dimension": "basic", "checks": { "basic": true }, "passed": true, "status": "passed", "checks_completed": 6, "duration_seconds": 0.531285, "transcript_sha256": "c809eed26dc56583f9385d7d4adca2d981d06a672dc1ccbc0ffc52aa228b86f7", "error": null }, { "seed": 17, "case_id": "json-types-and-overwrite", "dimension": "basic", "checks": { "basic": true }, "passed": true, "status": "passed", "checks_completed": 16, "duration_seconds": 0.542066, "transcript_sha256": "83f6377f4b57ac659e8162136107ca644158ee089c944892aa9817085e74bd60", "error": null }, { "seed": 17, "case_id": "reject-and-continue", "dimension": "validation", "checks": { "validation": true }, "passed": true, "status": "passed", "checks_completed": 10, "duration_seconds": 0.514736, "transcript_sha256": "d4e0c26837f54e0af94d4909e3cab21d86dedca8c8842d0daae9e7471198c2a0", "error": null }, { "seed": 17, "case_id": "empty-and-unicode-keys", "dimension": "validation", "checks": { "validation": true }, "passed": true, "status": "passed", "checks_completed": 4, "duration_seconds": 0.511955, "transcript_sha256": "e8abe374fc308ce0744e96d2920495b96bda3b026cec9b86fa055023e1e4e620", "error": null }, { "seed": 17, "case_id": "commit-batch", "dimension": "transactions", "checks": { "transactions": true }, "passed": true, "status": "passed", "checks_completed": 4, "duration_seconds": 0.506471, "transcript_sha256": "ac21f2f6bcc47e3c807fa4ed72300af0f5ef39809ba17f93423e37a0e177e718", "error": null }, { "seed": 17, "case_id": "rollback-batch", "dimension": "transactions", "checks": { "transactions": false }, "passed": false, "status": "failed", "checks_completed": 2, "duration_seconds": 0.497347, "transcript_sha256": "f6015865dce884224cf6ccccc9fbdd367baad17842dde00fbee476f06b1aab0d", "error": "response mismatch at request 3; expected {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}, got {\"found\": true, \"ok\": true, \"value\": \"must roll back\"}" }, { "seed": 17, "case_id": "cas-match-and-mismatch", "dimension": "compare_swap", "checks": { "compare_swap": true }, "passed": true, "status": "passed", "checks_completed": 7, "duration_seconds": 0.533444, "transcript_sha256": "ecd6148f6f387b4ef5351b877afd4cf27ea2a04c4253a34220fd78de5442d89f", "error": null }, { "seed": 17, "case_id": "cas-type-sensitivity", "dimension": "compare_swap", "checks": { "compare_swap": true }, "passed": true, "status": "passed", "checks_completed": 5, "duration_seconds": 0.498659, "transcript_sha256": "459a5b74588217d8abcf8e138eb559364a1d01d1f23e8c43ca0127ba28240258", "error": null }, { "seed": 17, "case_id": "restart-preserves-acknowledged-write", "dimension": "persistence", "checks": { "persistence": true }, "passed": true, "status": "passed", "checks_completed": 4, "duration_seconds": 1.499686, "transcript_sha256": "c97b82c67f26a1aa2fdc8ac57f50690dbbe13554d094059ffe6690a592df3a55", "error": null }, { "seed": 17, "case_id": "restart-preserves-transaction", "dimension": "persistence", "checks": { "persistence": true }, "passed": true, "status": "passed", "checks_completed": 3, "duration_seconds": 0.997884, "transcript_sha256": "2d1f704df309d49f8e6e89e674581af2cf81aa22fdde633058e2f2d7b8c8a430", "error": null }, { "seed": 17, "case_id": "sigkill-after-acknowledgement", "dimension": "crash_recovery", "checks": { "crash_recovery": true }, "passed": true, "status": "passed", "checks_completed": 2, "duration_seconds": 0.99713, "transcript_sha256": "544c6be916371ed5a0e901394c5e293733ca84e0ad20079a952f9c4b2821a728", "error": null }, { "seed": 17, "case_id": "sigkill-after-atomic-batch", "dimension": "crash_recovery", "checks": { "crash_recovery": true }, "passed": true, "status": "passed", "checks_completed": 3, "duration_seconds": 1.008529, "transcript_sha256": "2d1f704df309d49f8e6e89e674581af2cf81aa22fdde633058e2f2d7b8c8a430", "error": null }, { "seed": 17, "case_id": "restart-preserves-cas", "dimension": "persistence", "checks": { "persistence": true }, "passed": true, "status": "passed", "checks_completed": 3, "duration_seconds": 1.060491, "transcript_sha256": "149c04fd1366e77603e631349740ae38bdff40aa69f6eb4c738979eb13bf3f49", "error": null }, { "seed": 17, "case_id": "sigkill-after-delete", "dimension": "crash_recovery", "checks": { "crash_recovery": true }, "passed": true, "status": "passed", "checks_completed": 3, "duration_seconds": 1.694327, "transcript_sha256": "b3dba7f28e6629bb22f49b0439e0f74580ca3263dd98e16061ac9a8f680b5df7", "error": null }, { "seed": 17, "case_id": "seeded-state-machine", "dimension": "basic", "checks": { "basic": true }, "passed": true, "status": "passed", "checks_completed": 52, "duration_seconds": 0.830685, "transcript_sha256": "2a7e5ad4bedce67faf4d53f289c3a1a7304345a0167b287418fdf7dd411030f8", "error": null } ], "evaluation_seconds": 12.23358, "agent_cost_usd": null, "agent_tokens": null } }, { "name": "cas-always-wins", "target_dimension": "compare_swap", "killed": true, "valid": true, "score": 0.85, "failing_cases": [ "cas-match-and-mismatch", "cas-type-sensitivity" ], "candidate_sha256": "eb503c0ba87888db36fea991fc389223855c3e5d4c75ad9979a92e6109846b80", "evaluation": { "schema_version": "evalarc.evaluation.v2", "created_at": "2026-09-14T01:30:39.354373+00:00", "evalarc_version": "0.2.0", "task": { "id": "durable-kv", "version": "0.1.0", "domain": "coding", "split": "public-development" }, "candidate_sha256": "eb503c0ba87888db36fea991fc389223855c3e5d4c75ad9979a92e6109846b80", "grader_sha256": "52caa3136434569f96097860bc9bf85eb7ada5f7f9890c0540db81afb441871d", "cases_sha256": "842a16a1e65700cde92ee5f5aaaa10d494a9fc9d71bd2d99fb68147b19aad63a", "runtime": { "backend": "docker", "image": "python:3.12-slim", "image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea", "command": [ "{python}", "-I", "-B", "main.py", "{state}/store.db" ], "response_timeout_seconds": 10.0, "session_output_limit_bytes": 1048576, "python": "3.12.3", "platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39" }, "seeds": [ 17 ], "valid": true, "status": "failed", "score": 0.85, "resolved": false, "dimensions": { "basic": { "passed": 3, "total": 3, "assessed": 3, "weight": 0.25, "score": 1.0 }, "validation": { "passed": 2, "total": 2, "assessed": 2, "weight": 0.15, "score": 1.0 }, "transactions": { "passed": 2, "total": 2, "assessed": 2, "weight": 0.2, "score": 1.0 }, "compare_swap": { "passed": 0, "total": 2, "assessed": 2, "weight": 0.15, "score": 0.0 }, "persistence": { "passed": 3, "total": 3, "assessed": 3, "weight": 0.15, "score": 1.0 }, "crash_recovery": { "passed": 3, "total": 3, "assessed": 3, "weight": 0.1, "score": 1.0 } }, "cases": [ { "seed": 17, "case_id": "read-write-delete", "dimension": "basic", "checks": { "basic": true }, "passed": true, "status": "passed", "checks_completed": 6, "duration_seconds": 0.601743, "transcript_sha256": "c809eed26dc56583f9385d7d4adca2d981d06a672dc1ccbc0ffc52aa228b86f7", "error": null }, { "seed": 17, "case_id": "json-types-and-overwrite", "dimension": "basic", "checks": { "basic": true }, "passed": true, "status": "passed", "checks_completed": 16, "duration_seconds": 0.666199, "transcript_sha256": "83f6377f4b57ac659e8162136107ca644158ee089c944892aa9817085e74bd60", "error": null }, { "seed": 17, "case_id": "reject-and-continue", "dimension": "validation", "checks": { "validation": true }, "passed": true, "status": "passed", "checks_completed": 10, "duration_seconds": 0.582939, "transcript_sha256": "d4e0c26837f54e0af94d4909e3cab21d86dedca8c8842d0daae9e7471198c2a0", "error": null }, { "seed": 17, "case_id": "empty-and-unicode-keys", "dimension": "validation", "checks": { "validation": true }, "passed": true, "status": "passed", "checks_completed": 4, "duration_seconds": 0.591822, "transcript_sha256": "e8abe374fc308ce0744e96d2920495b96bda3b026cec9b86fa055023e1e4e620", "error": null }, { "seed": 17, "case_id": "commit-batch", "dimension": "transactions", "checks": { "transactions": true }, "passed": true, "status": "passed", "checks_completed": 4, "duration_seconds": 0.550826, "transcript_sha256": "ac21f2f6bcc47e3c807fa4ed72300af0f5ef39809ba17f93423e37a0e177e718", "error": null }, { "seed": 17, "case_id": "rollback-batch", "dimension": "transactions", "checks": { "transactions": true }, "passed": true, "status": "passed", "checks_completed": 7, "duration_seconds": 0.558259, "transcript_sha256": "1786c093ba29dcad562531a44cdbe5fddbed9d3f4632c1c6ddfe736baa240f35", "error": null }, { "seed": 17, "case_id": "cas-match-and-mismatch", "dimension": "compare_swap", "checks": { "compare_swap": false }, "passed": false, "status": "failed", "checks_completed": 1, "duration_seconds": 0.557099, "transcript_sha256": "71199e89272547e9c1a1961a85a650b93c30e2f848912b6be28ea262907eda56", "error": "response mismatch at request 2; expected {\"ok\": true, \"swapped\": false}, got {\"ok\": true, \"swapped\": true}" }, { "seed": 17, "case_id": "cas-type-sensitivity", "dimension": "compare_swap", "checks": { "compare_swap": false }, "passed": false, "status": "failed", "checks_completed": 1, "duration_seconds": 0.565552, "transcript_sha256": "70231a539612a6973a06489411646982ea651ca401223316f7f670e6560a18bf", "error": "response mismatch at request 2; expected {\"ok\": true, \"swapped\": false}, got {\"ok\": true, \"swapped\": true}" }, { "seed": 17, "case_id": "restart-preserves-acknowledged-write", "dimension": "persistence", "checks": { "persistence": true }, "passed": true, "status": "passed", "checks_completed": 4, "duration_seconds": 1.660789, "transcript_sha256": "c97b82c67f26a1aa2fdc8ac57f50690dbbe13554d094059ffe6690a592df3a55", "error": null }, { "seed": 17, "case_id": "restart-preserves-transaction", "dimension": "persistence", "checks": { "persistence": true }, "passed": true, "status": "passed", "checks_completed": 3, "duration_seconds": 1.16438, "transcript_sha256": "2d1f704df309d49f8e6e89e674581af2cf81aa22fdde633058e2f2d7b8c8a430", "error": null }, { "seed": 17, "case_id": "sigkill-after-acknowledgement", "dimension": "crash_recovery", "checks": { "crash_recovery": true }, "passed": true, "status": "passed", "checks_completed": 2, "duration_seconds": 1.396278, "transcript_sha256": "544c6be916371ed5a0e901394c5e293733ca84e0ad20079a952f9c4b2821a728", "error": null }, { "seed": 17, "case_id": "sigkill-after-atomic-batch", "dimension": "crash_recovery", "checks": { "crash_recovery": true }, "passed": true, "status": "passed", "checks_completed": 3, "duration_seconds": 1.131043, "transcript_sha256": "2d1f704df309d49f8e6e89e674581af2cf81aa22fdde633058e2f2d7b8c8a430", "error": null }, { "seed": 17, "case_id": "restart-preserves-cas", "dimension": "persistence", "checks": { "persistence": true }, "passed": true, "status": "passed", "checks_completed": 3, "duration_seconds": 1.129017, "transcript_sha256": "149c04fd1366e77603e631349740ae38bdff40aa69f6eb4c738979eb13bf3f49", "error": null }, { "seed": 17, "case_id": "sigkill-after-delete", "dimension": "crash_recovery", "checks": { "crash_recovery": true }, "passed": true, "status": "passed", "checks_completed": 3, "duration_seconds": 1.646182, "transcript_sha256": "b3dba7f28e6629bb22f49b0439e0f74580ca3263dd98e16061ac9a8f680b5df7", "error": null }, { "seed": 17, "case_id": "seeded-state-machine", "dimension": "basic", "checks": { "basic": true }, "passed": true, "status": "passed", "checks_completed": 52, "duration_seconds": 0.753312, "transcript_sha256": "2a7e5ad4bedce67faf4d53f289c3a1a7304345a0167b287418fdf7dd411030f8", "error": null } ], "evaluation_seconds": 13.566574, "agent_cost_usd": null, "agent_tokens": null } }, { "name": "boolean-equals-one", "target_dimension": "compare_swap", "killed": true, "valid": true, "score": 0.925, "failing_cases": [ "cas-type-sensitivity" ], "candidate_sha256": "8b3b2b2e4d41f2b0cec36bdc4aef42dad3969f9d81b9a0a0d6cde665d538aac3", "evaluation": { "schema_version": "evalarc.evaluation.v2", "created_at": "2026-09-14T01:30:52.200917+00:00", "evalarc_version": "0.2.0", "task": { "id": "durable-kv", "version": "0.1.0", "domain": "coding", "split": "public-development" }, "candidate_sha256": "8b3b2b2e4d41f2b0cec36bdc4aef42dad3969f9d81b9a0a0d6cde665d538aac3", "grader_sha256": "52caa3136434569f96097860bc9bf85eb7ada5f7f9890c0540db81afb441871d", "cases_sha256": "842a16a1e65700cde92ee5f5aaaa10d494a9fc9d71bd2d99fb68147b19aad63a", "runtime": { "backend": "docker", "image": "python:3.12-slim", "image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea", "command": [ "{python}", "-I", "-B", "main.py", "{state}/store.db" ], "response_timeout_seconds": 10.0, "session_output_limit_bytes": 1048576, "python": "3.12.3", "platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39" }, "seeds": [ 17 ], "valid": true, "status": "failed", "score": 0.925, "resolved": false, "dimensions": { "basic": { "passed": 3, "total": 3, "assessed": 3, "weight": 0.25, "score": 1.0 }, "validation": { "passed": 2, "total": 2, "assessed": 2, "weight": 0.15, "score": 1.0 }, "transactions": { "passed": 2, "total": 2, "assessed": 2, "weight": 0.2, "score": 1.0 }, "compare_swap": { "passed": 1, "total": 2, "assessed": 2, "weight": 0.15, "score": 0.5 }, "persistence": { "passed": 3, "total": 3, "assessed": 3, "weight": 0.15, "score": 1.0 }, "crash_recovery": { "passed": 3, "total": 3, "assessed": 3, "weight": 0.1, "score": 1.0 } }, "cases": [ { "seed": 17, "case_id": "read-write-delete", "dimension": "basic", "checks": { "basic": true }, "passed": true, "status": "passed", "checks_completed": 6, "duration_seconds": 0.556055, "transcript_sha256": "c809eed26dc56583f9385d7d4adca2d981d06a672dc1ccbc0ffc52aa228b86f7", "error": null }, { "seed": 17, "case_id": "json-types-and-overwrite", "dimension": "basic", "checks": { "basic": true }, "passed": true, "status": "passed", "checks_completed": 16, "duration_seconds": 0.602162, "transcript_sha256": "83f6377f4b57ac659e8162136107ca644158ee089c944892aa9817085e74bd60", "error": null }, { "seed": 17, "case_id": "reject-and-continue", "dimension": "validation", "checks": { "validation": true }, "passed": true, "status": "passed", "checks_completed": 10, "duration_seconds": 0.538341, "transcript_sha256": "d4e0c26837f54e0af94d4909e3cab21d86dedca8c8842d0daae9e7471198c2a0", "error": null }, { "seed": 17, "case_id": "empty-and-unicode-keys", "dimension": "validation", "checks": { "validation": true }, "passed": true, "status": "passed", "checks_completed": 4, "duration_seconds": 0.550951, "transcript_sha256": "e8abe374fc308ce0744e96d2920495b96bda3b026cec9b86fa055023e1e4e620", "error": null }, { "seed": 17, "case_id": "commit-batch", "dimension": "transactions", "checks": { "transactions": true }, "passed": true, "status": "passed", "checks_completed": 4, "duration_seconds": 0.551653, "transcript_sha256": "ac21f2f6bcc47e3c807fa4ed72300af0f5ef39809ba17f93423e37a0e177e718", "error": null }, { "seed": 17, "case_id": "rollback-batch", "dimension": "transactions", "checks": { "transactions": true }, "passed": true, "status": "passed", "checks_completed": 7, "duration_seconds": 0.559257, "transcript_sha256": "1786c093ba29dcad562531a44cdbe5fddbed9d3f4632c1c6ddfe736baa240f35", "error": null }, { "seed": 17, "case_id": "cas-match-and-mismatch", "dimension": "compare_swap", "checks": { "compare_swap": true }, "passed": true, "status": "passed", "checks_completed": 7, "duration_seconds": 0.552929, "transcript_sha256": "ecd6148f6f387b4ef5351b877afd4cf27ea2a04c4253a34220fd78de5442d89f", "error": null }, { "seed": 17, "case_id": "cas-type-sensitivity", "dimension": "compare_swap", "checks": { "compare_swap": false }, "passed": false, "status": "failed", "checks_completed": 1, "duration_seconds": 0.57604, "transcript_sha256": "70231a539612a6973a06489411646982ea651ca401223316f7f670e6560a18bf", "error": "response mismatch at request 2; expected {\"ok\": true, \"swapped\": false}, got {\"ok\": true, \"swapped\": true}" }, { "seed": 17, "case_id": "restart-preserves-acknowledged-write", "dimension": "persistence", "checks": { "persistence": true }, "passed": true, "status": "passed", "checks_completed": 4, "duration_seconds": 1.639025, "transcript_sha256": "c97b82c67f26a1aa2fdc8ac57f50690dbbe13554d094059ffe6690a592df3a55", "error": null }, { "seed": 17, "case_id": "restart-preserves-transaction", "dimension": "persistence", "checks": { "persistence": true }, "passed": true, "status": "passed", "checks_completed": 3, "duration_seconds": 1.062938, "transcript_sha256": "2d1f704df309d49f8e6e89e674581af2cf81aa22fdde633058e2f2d7b8c8a430", "error": null }, { "seed": 17, "case_id": "sigkill-after-acknowledgement", "dimension": "crash_recovery", "checks": { "crash_recovery": true }, "passed": true, "status": "passed", "checks_completed": 2, "duration_seconds": 1.049137, "transcript_sha256": "544c6be916371ed5a0e901394c5e293733ca84e0ad20079a952f9c4b2821a728", "error": null }, { "seed": 17, "case_id": "sigkill-after-atomic-batch", "dimension": "crash_recovery", "checks": { "crash_recovery": true }, "passed": true, "status": "passed", "checks_completed": 3, "duration_seconds": 1.085374, "transcript_sha256": "2d1f704df309d49f8e6e89e674581af2cf81aa22fdde633058e2f2d7b8c8a430", "error": null }, { "seed": 17, "case_id": "restart-preserves-cas", "dimension": "persistence", "checks": { "persistence": true }, "passed": true, "status": "passed", "checks_completed": 3, "duration_seconds": 1.08237, "transcript_sha256": "149c04fd1366e77603e631349740ae38bdff40aa69f6eb4c738979eb13bf3f49", "error": null }, { "seed": 17, "case_id": "sigkill-after-delete", "dimension": "crash_recovery", "checks": { "crash_recovery": true }, "passed": true, "status": "passed", "checks_completed": 3, "duration_seconds": 1.6509, "transcript_sha256": "b3dba7f28e6629bb22f49b0439e0f74580ca3263dd98e16061ac9a8f680b5df7", "error": null }, { "seed": 17, "case_id": "seeded-state-machine", "dimension": "basic", "checks": { "basic": true }, "passed": true, "status": "passed", "checks_completed": 52, "duration_seconds": 0.780467, "transcript_sha256": "2a7e5ad4bedce67faf4d53f289c3a1a7304345a0167b287418fdf7dd411030f8", "error": null } ], "evaluation_seconds": 12.84625, "agent_cost_usd": null, "agent_tokens": null } }, { "name": "delete-noop", "target_dimension": "basic", "killed": true, "valid": true, "score": 0.65, "failing_cases": [ "read-write-delete", "seeded-state-machine" ], "candidate_sha256": "2c81d23c574991d7a80b45df1b79ada77309e930e73b8a41a8d0a1e4440625f8", "evaluation": { "schema_version": "evalarc.evaluation.v2", "created_at": "2026-09-14T01:31:04.674524+00:00", "evalarc_version": "0.2.0", "task": { "id": "durable-kv", "version": "0.1.0", "domain": "coding", "split": "public-development" }, "candidate_sha256": "2c81d23c574991d7a80b45df1b79ada77309e930e73b8a41a8d0a1e4440625f8", "grader_sha256": "52caa3136434569f96097860bc9bf85eb7ada5f7f9890c0540db81afb441871d", "cases_sha256": "842a16a1e65700cde92ee5f5aaaa10d494a9fc9d71bd2d99fb68147b19aad63a", "runtime": { "backend": "docker", "image": "python:3.12-slim", "image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea", "command": [ "{python}", "-I", "-B", "main.py", "{state}/store.db" ], "response_timeout_seconds": 10.0, "session_output_limit_bytes": 1048576, "python": "3.12.3", "platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39" }, "seeds": [ 17 ], "valid": true, "status": "failed", "score": 0.65, "resolved": false, "dimensions": { "basic": { "passed": 1, "total": 3, "assessed": 3, "weight": 0.25, "score": 0.3333333333333333 }, "validation": { "passed": 2, "total": 2, "assessed": 2, "weight": 0.15, "score": 1.0 }, "transactions": { "passed": 1, "total": 2, "assessed": 2, "weight": 0.2, "score": 0.5 }, "compare_swap": { "passed": 2, "total": 2, "assessed": 2, "weight": 0.15, "score": 1.0 }, "persistence": { "passed": 2, "total": 3, "assessed": 3, "weight": 0.15, "score": 0.6666666666666666 }, "crash_recovery": { "passed": 2, "total": 3, "assessed": 3, "weight": 0.1, "score": 0.6666666666666666 } }, "cases": [ { "seed": 17, "case_id": "read-write-delete", "dimension": "basic", "checks": { "basic": false }, "passed": false, "status": "failed", "checks_completed": 4, "duration_seconds": 0.557506, "transcript_sha256": "4122b55bc205e4703a3a9f62a5397de448a833f424e54edc914bbbf5465c70d1", "error": "response mismatch at request 5; expected {\"found\": false, \"ok\": true}, got {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}" }, { "seed": 17, "case_id": "json-types-and-overwrite", "dimension": "basic", "checks": { "basic": true }, "passed": true, "status": "passed", "checks_completed": 16, "duration_seconds": 0.623245, "transcript_sha256": "83f6377f4b57ac659e8162136107ca644158ee089c944892aa9817085e74bd60", "error": null }, { "seed": 17, "case_id": "reject-and-continue", "dimension": "validation", "checks": { "validation": true }, "passed": true, "status": "passed", "checks_completed": 10, "duration_seconds": 0.55319, "transcript_sha256": "d4e0c26837f54e0af94d4909e3cab21d86dedca8c8842d0daae9e7471198c2a0", "error": null }, { "seed": 17, "case_id": "empty-and-unicode-keys", "dimension": "validation", "checks": { "validation": true }, "passed": true, "status": "passed", "checks_completed": 4, "duration_seconds": 0.527258, "transcript_sha256": "e8abe374fc308ce0744e96d2920495b96bda3b026cec9b86fa055023e1e4e620", "error": null }, { "seed": 17, "case_id": "commit-batch", "dimension": "transactions", "checks": { "transactions": false }, "passed": false, "status": "failed", "checks_completed": 1, "duration_seconds": 0.533816, "transcript_sha256": "da7df3388998e4f6fd4743177baa1ddcd8625d54fded8d5c6cfbdf813d50cdba", "error": "response mismatch at request 2; expected {\"found\": false, \"ok\": true}, got {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}" }, { "seed": 17, "case_id": "rollback-batch", "dimension": "transactions", "checks": { "transactions": true }, "passed": true, "status": "passed", "checks_completed": 7, "duration_seconds": 0.571187, "transcript_sha256": "1786c093ba29dcad562531a44cdbe5fddbed9d3f4632c1c6ddfe736baa240f35", "error": null }, { "seed": 17, "case_id": "cas-match-and-mismatch", "dimension": "compare_swap", "checks": { "compare_swap": true }, "passed": true, "status": "passed", "checks_completed": 7, "duration_seconds": 0.53584, "transcript_sha256": "ecd6148f6f387b4ef5351b877afd4cf27ea2a04c4253a34220fd78de5442d89f", "error": null }, { "seed": 17, "case_id": "cas-type-sensitivity", "dimension": "compare_swap", "checks": { "compare_swap": true }, "passed": true, "status": "passed", "checks_completed": 5, "duration_seconds": 0.568142, "transcript_sha256": "459a5b74588217d8abcf8e138eb559364a1d01d1f23e8c43ca0127ba28240258", "error": null }, { "seed": 17, "case_id": "restart-preserves-acknowledged-write", "dimension": "persistence", "checks": { "persistence": false }, "passed": false, "status": "failed", "checks_completed": 3, "duration_seconds": 1.643947, "transcript_sha256": "0ae3e20082ef3346a8027b09f61e373e1c733234603e3e12d5e61853d6df9c8f", "error": "response mismatch at request 4; expected {\"found\": false, \"ok\": true}, got {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}" }, { "seed": 17, "case_id": "restart-preserves-transaction", "dimension": "persistence", "checks": { "persistence": true }, "passed": true, "status": "passed", "checks_completed": 3, "duration_seconds": 1.090287, "transcript_sha256": "2d1f704df309d49f8e6e89e674581af2cf81aa22fdde633058e2f2d7b8c8a430", "error": null }, { "seed": 17, "case_id": "sigkill-after-acknowledgement", "dimension": "crash_recovery", "checks": { "crash_recovery": true }, "passed": true, "status": "passed", "checks_completed": 2, "duration_seconds": 1.113412, "transcript_sha256": "544c6be916371ed5a0e901394c5e293733ca84e0ad20079a952f9c4b2821a728", "error": null }, { "seed": 17, "case_id": "sigkill-after-atomic-batch", "dimension": "crash_recovery", "checks": { "crash_recovery": true }, "passed": true, "status": "passed", "checks_completed": 3, "duration_seconds": 1.077148, "transcript_sha256": "2d1f704df309d49f8e6e89e674581af2cf81aa22fdde633058e2f2d7b8c8a430", "error": null }, { "seed": 17, "case_id": "restart-preserves-cas", "dimension": "persistence", "checks": { "persistence": true }, "passed": true, "status": "passed", "checks_completed": 3, "duration_seconds": 1.085685, "transcript_sha256": "149c04fd1366e77603e631349740ae38bdff40aa69f6eb4c738979eb13bf3f49", "error": null }, { "seed": 17, "case_id": "sigkill-after-delete", "dimension": "crash_recovery", "checks": { "crash_recovery": false }, "passed": false, "status": "failed", "checks_completed": 2, "duration_seconds": 1.429231, "transcript_sha256": "679e7ab1f7f833fb12551b29af4fdc1ad8e1051810e594a4da85fdc3420b8c42", "error": "response mismatch at request 3; expected {\"found\": false, \"ok\": true}, got {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}" }, { "seed": 17, "case_id": "seeded-state-machine", "dimension": "basic", "checks": { "basic": false }, "passed": false, "status": "failed", "checks_completed": 8, "duration_seconds": 0.556806, "transcript_sha256": "0132892f9b4bac6264eb6ef2dae9da8b7bac7a6a158b56357f1319c853615de7", "error": "response mismatch at request 9; expected {\"found\": false, \"ok\": true}, got {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}" } ], "evaluation_seconds": 12.473283, "agent_cost_usd": null, "agent_tokens": null } }, { "name": "accept-nonstring-keys", "target_dimension": "validation", "killed": true, "valid": true, "score": 0.925, "failing_cases": [ "reject-and-continue" ], "candidate_sha256": "1c3846e123f39e86cf3302b5548674c429e4c740e178563ad9f9457ab784f5a4", "evaluation": { "schema_version": "evalarc.evaluation.v2", "created_at": "2026-09-14T01:31:16.266965+00:00", "evalarc_version": "0.2.0", "task": { "id": "durable-kv", "version": "0.1.0", "domain": "coding", "split": "public-development" }, "candidate_sha256": "1c3846e123f39e86cf3302b5548674c429e4c740e178563ad9f9457ab784f5a4", "grader_sha256": "52caa3136434569f96097860bc9bf85eb7ada5f7f9890c0540db81afb441871d", "cases_sha256": "842a16a1e65700cde92ee5f5aaaa10d494a9fc9d71bd2d99fb68147b19aad63a", "runtime": { "backend": "docker", "image": "python:3.12-slim", "image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea", "command": [ "{python}", "-I", "-B", "main.py", "{state}/store.db" ], "response_timeout_seconds": 10.0, "session_output_limit_bytes": 1048576, "python": "3.12.3", "platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39" }, "seeds": [ 17 ], "valid": true, "status": "failed", "score": 0.925, "resolved": false, "dimensions": { "basic": { "passed": 3, "total": 3, "assessed": 3, "weight": 0.25, "score": 1.0 }, "validation": { "passed": 1, "total": 2, "assessed": 2, "weight": 0.15, "score": 0.5 }, "transactions": { "passed": 2, "total": 2, "assessed": 2, "weight": 0.2, "score": 1.0 }, "compare_swap": { "passed": 2, "total": 2, "assessed": 2, "weight": 0.15, "score": 1.0 }, "persistence": { "passed": 3, "total": 3, "assessed": 3, "weight": 0.15, "score": 1.0 }, "crash_recovery": { "passed": 3, "total": 3, "assessed": 3, "weight": 0.1, "score": 1.0 } }, "cases": [ { "seed": 17, "case_id": "read-write-delete", "dimension": "basic", "checks": { "basic": true }, "passed": true, "status": "passed", "checks_completed": 6, "duration_seconds": 0.487851, "transcript_sha256": "c809eed26dc56583f9385d7d4adca2d981d06a672dc1ccbc0ffc52aa228b86f7", "error": null }, { "seed": 17, "case_id": "json-types-and-overwrite", "dimension": "basic", "checks": { "basic": true }, "passed": true, "status": "passed", "checks_completed": 16, "duration_seconds": 0.54511, "transcript_sha256": "83f6377f4b57ac659e8162136107ca644158ee089c944892aa9817085e74bd60", "error": null }, { "seed": 17, "case_id": "reject-and-continue", "dimension": "validation", "checks": { "validation": false }, "passed": false, "status": "failed", "checks_completed": 4, "duration_seconds": 0.519073, "transcript_sha256": "dc07c86f871afd6b2f208e9751bdd459e52368d54d008ca0622a8edd61a7008b", "error": "response mismatch at request 5; expected {\"error\": \"invalid_request\", \"ok\": false}, got {\"ok\": true}" }, { "seed": 17, "case_id": "empty-and-unicode-keys", "dimension": "validation", "checks": { "validation": true }, "passed": true, "status": "passed", "checks_completed": 4, "duration_seconds": 0.485221, "transcript_sha256": "e8abe374fc308ce0744e96d2920495b96bda3b026cec9b86fa055023e1e4e620", "error": null }, { "seed": 17, "case_id": "commit-batch", "dimension": "transactions", "checks": { "transactions": true }, "passed": true, "status": "passed", "checks_completed": 4, "duration_seconds": 0.509709, "transcript_sha256": "ac21f2f6bcc47e3c807fa4ed72300af0f5ef39809ba17f93423e37a0e177e718", "error": null }, { "seed": 17, "case_id": "rollback-batch", "dimension": "transactions", "checks": { "transactions": true }, "passed": true, "status": "passed", "checks_completed": 7, "duration_seconds": 0.509348, "transcript_sha256": "1786c093ba29dcad562531a44cdbe5fddbed9d3f4632c1c6ddfe736baa240f35", "error": null }, { "seed": 17, "case_id": "cas-match-and-mismatch", "dimension": "compare_swap", "checks": { "compare_swap": true }, "passed": true, "status": "passed", "checks_completed": 7, "duration_seconds": 0.515143, "transcript_sha256": "ecd6148f6f387b4ef5351b877afd4cf27ea2a04c4253a34220fd78de5442d89f", "error": null }, { "seed": 17, "case_id": "cas-type-sensitivity", "dimension": "compare_swap", "checks": { "compare_swap": true }, "passed": true, "status": "passed", "checks_completed": 5, "duration_seconds": 0.501223, "transcript_sha256": "459a5b74588217d8abcf8e138eb559364a1d01d1f23e8c43ca0127ba28240258", "error": null }, { "seed": 17, "case_id": "restart-preserves-acknowledged-write", "dimension": "persistence", "checks": { "persistence": true }, "passed": true, "status": "passed", "checks_completed": 4, "duration_seconds": 1.484043, "transcript_sha256": "c97b82c67f26a1aa2fdc8ac57f50690dbbe13554d094059ffe6690a592df3a55", "error": null }, { "seed": 17, "case_id": "restart-preserves-transaction", "dimension": "persistence", "checks": { "persistence": true }, "passed": true, "status": "passed", "checks_completed": 3, "duration_seconds": 0.98699, "transcript_sha256": "2d1f704df309d49f8e6e89e674581af2cf81aa22fdde633058e2f2d7b8c8a430", "error": null }, { "seed": 17, "case_id": "sigkill-after-acknowledgement", "dimension": "crash_recovery", "checks": { "crash_recovery": true }, "passed": true, "status": "passed", "checks_completed": 2, "duration_seconds": 0.981747, "transcript_sha256": "544c6be916371ed5a0e901394c5e293733ca84e0ad20079a952f9c4b2821a728", "error": null }, { "seed": 17, "case_id": "sigkill-after-atomic-batch", "dimension": "crash_recovery", "checks": { "crash_recovery": true }, "passed": true, "status": "passed", "checks_completed": 3, "duration_seconds": 0.987597, "transcript_sha256": "2d1f704df309d49f8e6e89e674581af2cf81aa22fdde633058e2f2d7b8c8a430", "error": null }, { "seed": 17, "case_id": "restart-preserves-cas", "dimension": "persistence", "checks": { "persistence": true }, "passed": true, "status": "passed", "checks_completed": 3, "duration_seconds": 0.981076, "transcript_sha256": "149c04fd1366e77603e631349740ae38bdff40aa69f6eb4c738979eb13bf3f49", "error": null }, { "seed": 17, "case_id": "sigkill-after-delete", "dimension": "crash_recovery", "checks": { "crash_recovery": true }, "passed": true, "status": "passed", "checks_completed": 3, "duration_seconds": 1.437775, "transcript_sha256": "b3dba7f28e6629bb22f49b0439e0f74580ca3263dd98e16061ac9a8f680b5df7", "error": null }, { "seed": 17, "case_id": "seeded-state-machine", "dimension": "basic", "checks": { "basic": true }, "passed": true, "status": "passed", "checks_completed": 52, "duration_seconds": 0.653102, "transcript_sha256": "2a7e5ad4bedce67faf4d53f289c3a1a7304345a0167b287418fdf7dd411030f8", "error": null } ], "evaluation_seconds": 11.592179, "agent_cost_usd": null, "agent_tokens": null } }, { "name": "commit-on-exit", "target_dimension": "crash_recovery", "killed": true, "valid": true, "score": 0.9, "failing_cases": [ "sigkill-after-acknowledgement", "sigkill-after-atomic-batch", "sigkill-after-delete" ], "candidate_sha256": "fce3eb722763136ecc184b4cc5fd02f80623615ed44a438caf69f561e4cc438f", "evaluation": { "schema_version": "evalarc.evaluation.v2", "created_at": "2026-09-14T01:31:27.622867+00:00", "evalarc_version": "0.2.0", "task": { "id": "durable-kv", "version": "0.1.0", "domain": "coding", "split": "public-development" }, "candidate_sha256": "fce3eb722763136ecc184b4cc5fd02f80623615ed44a438caf69f561e4cc438f", "grader_sha256": "52caa3136434569f96097860bc9bf85eb7ada5f7f9890c0540db81afb441871d", "cases_sha256": "842a16a1e65700cde92ee5f5aaaa10d494a9fc9d71bd2d99fb68147b19aad63a", "runtime": { "backend": "docker", "image": "python:3.12-slim", "image_id": "sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea", "command": [ "{python}", "-I", "-B", "main.py", "{state}/store.db" ], "response_timeout_seconds": 10.0, "session_output_limit_bytes": 1048576, "python": "3.12.3", "platform": "Linux-6.17.0-1020-aws-x86_64-with-glibc2.39" }, "seeds": [ 17 ], "valid": true, "status": "failed", "score": 0.9, "resolved": false, "dimensions": { "basic": { "passed": 3, "total": 3, "assessed": 3, "weight": 0.25, "score": 1.0 }, "validation": { "passed": 2, "total": 2, "assessed": 2, "weight": 0.15, "score": 1.0 }, "transactions": { "passed": 2, "total": 2, "assessed": 2, "weight": 0.2, "score": 1.0 }, "compare_swap": { "passed": 2, "total": 2, "assessed": 2, "weight": 0.15, "score": 1.0 }, "persistence": { "passed": 3, "total": 3, "assessed": 3, "weight": 0.15, "score": 1.0 }, "crash_recovery": { "passed": 0, "total": 3, "assessed": 3, "weight": 0.1, "score": 0.0 } }, "cases": [ { "seed": 17, "case_id": "read-write-delete", "dimension": "basic", "checks": { "basic": true }, "passed": true, "status": "passed", "checks_completed": 6, "duration_seconds": 0.498981, "transcript_sha256": "c809eed26dc56583f9385d7d4adca2d981d06a672dc1ccbc0ffc52aa228b86f7", "error": null }, { "seed": 17, "case_id": "json-types-and-overwrite", "dimension": "basic", "checks": { "basic": true }, "passed": true, "status": "passed", "checks_completed": 16, "duration_seconds": 0.510524, "transcript_sha256": "83f6377f4b57ac659e8162136107ca644158ee089c944892aa9817085e74bd60", "error": null }, { "seed": 17, "case_id": "reject-and-continue", "dimension": "validation", "checks": { "validation": true }, "passed": true, "status": "passed", "checks_completed": 10, "duration_seconds": 0.488127, "transcript_sha256": "d4e0c26837f54e0af94d4909e3cab21d86dedca8c8842d0daae9e7471198c2a0", "error": null }, { "seed": 17, "case_id": "empty-and-unicode-keys", "dimension": "validation", "checks": { "validation": true }, "passed": true, "status": "passed", "checks_completed": 4, "duration_seconds": 0.489482, "transcript_sha256": "e8abe374fc308ce0744e96d2920495b96bda3b026cec9b86fa055023e1e4e620", "error": null }, { "seed": 17, "case_id": "commit-batch", "dimension": "transactions", "checks": { "transactions": true }, "passed": true, "status": "passed", "checks_completed": 4, "duration_seconds": 0.499994, "transcript_sha256": "ac21f2f6bcc47e3c807fa4ed72300af0f5ef39809ba17f93423e37a0e177e718", "error": null }, { "seed": 17, "case_id": "rollback-batch", "dimension": "transactions", "checks": { "transactions": true }, "passed": true, "status": "passed", "checks_completed": 7, "duration_seconds": 0.533228, "transcript_sha256": "1786c093ba29dcad562531a44cdbe5fddbed9d3f4632c1c6ddfe736baa240f35", "error": null }, { "seed": 17, "case_id": "cas-match-and-mismatch", "dimension": "compare_swap", "checks": { "compare_swap": true }, "passed": true, "status": "passed", "checks_completed": 7, "duration_seconds": 0.502809, "transcript_sha256": "ecd6148f6f387b4ef5351b877afd4cf27ea2a04c4253a34220fd78de5442d89f", "error": null }, { "seed": 17, "case_id": "cas-type-sensitivity", "dimension": "compare_swap", "checks": { "compare_swap": true }, "passed": true, "status": "passed", "checks_completed": 5, "duration_seconds": 0.496033, "transcript_sha256": "459a5b74588217d8abcf8e138eb559364a1d01d1f23e8c43ca0127ba28240258", "error": null }, { "seed": 17, "case_id": "restart-preserves-acknowledged-write", "dimension": "persistence", "checks": { "persistence": true }, "passed": true, "status": "passed", "checks_completed": 4, "duration_seconds": 1.433264, "transcript_sha256": "c97b82c67f26a1aa2fdc8ac57f50690dbbe13554d094059ffe6690a592df3a55", "error": null }, { "seed": 17, "case_id": "restart-preserves-transaction", "dimension": "persistence", "checks": { "persistence": true }, "passed": true, "status": "passed", "checks_completed": 3, "duration_seconds": 0.995327, "transcript_sha256": "2d1f704df309d49f8e6e89e674581af2cf81aa22fdde633058e2f2d7b8c8a430", "error": null }, { "seed": 17, "case_id": "sigkill-after-acknowledgement", "dimension": "crash_recovery", "checks": { "crash_recovery": false }, "passed": false, "status": "failed", "checks_completed": 1, "duration_seconds": 0.998141, "transcript_sha256": "f62cc3fb4db094e24ad23141f7cc50908d24647156c4d2ba2386a2487c078e80", "error": "response mismatch at request 2; expected {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}, got {\"found\": false, \"ok\": true}" }, { "seed": 17, "case_id": "sigkill-after-atomic-batch", "dimension": "crash_recovery", "checks": { "crash_recovery": false }, "passed": false, "status": "failed", "checks_completed": 1, "duration_seconds": 0.978949, "transcript_sha256": "43cf3772fd4bd95486b1824b1d73c4e99859fb85088a7f62192d8325b4581b29", "error": "response mismatch at request 2; expected {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}, got {\"found\": false, \"ok\": true}" }, { "seed": 17, "case_id": "restart-preserves-cas", "dimension": "persistence", "checks": { "persistence": true }, "passed": true, "status": "passed", "checks_completed": 3, "duration_seconds": 0.939053, "transcript_sha256": "149c04fd1366e77603e631349740ae38bdff40aa69f6eb4c738979eb13bf3f49", "error": null }, { "seed": 17, "case_id": "sigkill-after-delete", "dimension": "crash_recovery", "checks": { "crash_recovery": false }, "passed": false, "status": "failed", "checks_completed": 2, "duration_seconds": 1.47289, "transcript_sha256": "679e7ab1f7f833fb12551b29af4fdc1ad8e1051810e594a4da85fdc3420b8c42", "error": "response mismatch at request 3; expected {\"found\": false, \"ok\": true}, got {\"found\": true, \"ok\": true, \"value\": {\"items\": [true, null, \"\\u03bb\\n\\u4e16\\u754c\"], \"version\": 5991}}" }, { "seed": 17, "case_id": "seeded-state-machine", "dimension": "basic", "checks": { "basic": true }, "passed": true, "status": "passed", "checks_completed": 52, "duration_seconds": 0.511465, "transcript_sha256": "2a7e5ad4bedce67faf4d53f289c3a1a7304345a0167b287418fdf7dd411030f8", "error": null } ], "evaluation_seconds": 11.355773, "agent_cost_usd": null, "agent_tokens": null } } ], "mutation_score": 1.0, "killed": 8, "total": 8, "passed": true, "interpretation": "Coverage of 8 declared behavioral fault models only. This is not a bound on reward hacking or a model benchmark." }