grammar-inference-engine/experiments/results/coarsen_fastapi.json
tobjend dd183f6241 feat: 4-codebase evaluation — conventions vs completions tradeoff
Added kotlinx.coroutines (1040 .kt files) and FastAPI (1129 .py files).

Key findings across 4 codebases:
- Coarsening trades per-package coverage for cross-package reach
- FastAPI: coverage IMPROVES (12.4% -> 20.7%), cross-pkg +15
- Coroutines: cross-pkg +15, real exception-handling patterns
- RAGSAK: cross-pkg +18, null-check conventions across 8 packages
- Flask: slight cross-pkg loss, rendering conventions still visible

The conventions vs completions tradeoff is real and measurable.
2026-07-12 01:14:09 +02:00

993 lines
No EOL
24 KiB
JSON

{
"name": "FastAPI",
"raw": {
"seq_count": 4811,
"alphabet_size": 1503,
"alphabet_sample": [
"",
"!r",
"\")",
"\")]",
"\", \"\",",
"\"Path must b",
"\"_{v}\"):",
"\"site",
"#",
"', '',",
"(",
"(\"do",
"(\"r",
"(\"s",
"(\"u",
"():",
"(1)",
"([\"",
"(ap",
"(banne",
"(content_f",
"(en_docs_path",
"(f\"U",
"(new_conte",
"(pro",
"(s",
"(scope",
"(upd",
"({\"lin",
"({code"
],
"k1": {
"contexts": 520,
"sore_successes": 53,
"total_methods": 4811,
"methods_in_good": 245,
"coverage": 5.1,
"cross_package_contexts": 88,
"elapsed": 0.005,
"top_groups": [
{
"context": "('_eps',)",
"label": "raw_k1",
"methods": 1154,
"unique": 1,
"unique_ratio": 0.001,
"alphabet_size": 0,
"sore": null,
"sore_success": false
},
{
"context": "('get',)",
"label": "raw_k1",
"methods": 1082,
"unique": 54,
"unique_ratio": 0.05,
"alphabet_size": 130,
"sore": null,
"sore_success": false
},
{
"context": "('TestClient',)",
"label": "raw_k1",
"methods": 408,
"unique": 33,
"unique_ratio": 0.081,
"alphabet_size": 26,
"sore": null,
"sore_success": false
},
{
"context": "('post',)",
"label": "raw_k1",
"methods": 196,
"unique": 16,
"unique_ratio": 0.082,
"alphabet_size": 15,
"sore": null,
"sore_success": false
},
{
"context": "('import_module',)",
"label": "raw_k1",
"methods": 151,
"unique": 4,
"unique_ratio": 0.026,
"alphabet_size": 6,
"sore": null,
"sore_success": false
},
{
"context": "('openapi',)",
"label": "raw_k1",
"methods": 90,
"unique": 4,
"unique_ratio": 0.044,
"alphabet_size": 10,
"sore": null,
"sore_success": false
},
{
"context": "('put',)",
"label": "raw_k1",
"methods": 72,
"unique": 4,
"unique_ratio": 0.056,
"alphabet_size": 4,
"sore": null,
"sore_success": false
},
{
"context": "('raises',)",
"label": "raw_k1",
"methods": 64,
"unique": 20,
"unique_ratio": 0.312,
"alphabet_size": 25,
"sore": null,
"sore_success": false
},
{
"context": "('FastAPI',)",
"label": "raw_k1",
"methods": 59,
"unique": 50,
"unique_ratio": 0.847,
"alphabet_size": 131,
"sore": null,
"sore_success": false
},
{
"context": "('HTTPException',)",
"label": "raw_k1",
"methods": 49,
"unique": 7,
"unique_ratio": 0.143,
"alphabet_size": 9,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('_eps',)",
"packages": 93
},
{
"context": "('get',)",
"packages": 86
},
{
"context": "('import_module',)",
"packages": 51
},
{
"context": "('post',)",
"packages": 29
},
{
"context": "('TestClient',)",
"packages": 25
},
{
"context": "('put',)",
"packages": 12
},
{
"context": "('HTTPException',)",
"packages": 10
},
{
"context": "('len',)",
"packages": 8
},
{
"context": "('openapi',)",
"packages": 8
},
{
"context": "('JSONResponse',)",
"packages": 7
}
]
},
"k2": {
"contexts": 536,
"sore_successes": 53,
"total_methods": 4811,
"methods_in_good": 245,
"coverage": 5.1,
"cross_package_contexts": 87,
"elapsed": 0.011,
"top_groups": [
{
"context": "('_eps',)",
"label": "raw_k2",
"methods": 1154,
"unique": 1,
"unique_ratio": 0.001,
"alphabet_size": 0,
"sore": null,
"sore_success": false
},
{
"context": "('get', 'get')",
"label": "raw_k2",
"methods": 1082,
"unique": 54,
"unique_ratio": 0.05,
"alphabet_size": 130,
"sore": null,
"sore_success": false
},
{
"context": "('TestClient', 'TestClient')",
"label": "raw_k2",
"methods": 408,
"unique": 33,
"unique_ratio": 0.081,
"alphabet_size": 26,
"sore": null,
"sore_success": false
},
{
"context": "('post', 'post')",
"label": "raw_k2",
"methods": 196,
"unique": 16,
"unique_ratio": 0.082,
"alphabet_size": 15,
"sore": null,
"sore_success": false
},
{
"context": "('import_module', 'import_module')",
"label": "raw_k2",
"methods": 151,
"unique": 4,
"unique_ratio": 0.026,
"alphabet_size": 6,
"sore": null,
"sore_success": false
},
{
"context": "('openapi', 'openapi')",
"label": "raw_k2",
"methods": 90,
"unique": 4,
"unique_ratio": 0.044,
"alphabet_size": 10,
"sore": null,
"sore_success": false
},
{
"context": "('put', 'put')",
"label": "raw_k2",
"methods": 72,
"unique": 4,
"unique_ratio": 0.056,
"alphabet_size": 4,
"sore": null,
"sore_success": false
},
{
"context": "('raises', 'raises')",
"label": "raw_k2",
"methods": 64,
"unique": 20,
"unique_ratio": 0.312,
"alphabet_size": 25,
"sore": null,
"sore_success": false
},
{
"context": "('FastAPI', 'FastAPI')",
"label": "raw_k2",
"methods": 59,
"unique": 50,
"unique_ratio": 0.847,
"alphabet_size": 131,
"sore": null,
"sore_success": false
},
{
"context": "('HTTPException', 'HTTPException')",
"label": "raw_k2",
"methods": 49,
"unique": 7,
"unique_ratio": 0.143,
"alphabet_size": 9,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('_eps',)",
"packages": 93
},
{
"context": "('get', 'get')",
"packages": 86
},
{
"context": "('import_module', 'import_module')",
"packages": 51
},
{
"context": "('post', 'post')",
"packages": 29
},
{
"context": "('TestClient', 'TestClient')",
"packages": 25
},
{
"context": "('put', 'put')",
"packages": 12
},
{
"context": "('HTTPException', 'HTTPException')",
"packages": 10
},
{
"context": "('len', 'len')",
"packages": 8
},
{
"context": "('openapi', 'openapi')",
"packages": 8
},
{
"context": "('JSONResponse', 'JSONResponse')",
"packages": 7
}
]
},
"k3": {
"contexts": 796,
"sore_successes": 92,
"total_methods": 4811,
"methods_in_good": 596,
"coverage": 12.4,
"cross_package_contexts": 82,
"elapsed": 0.005,
"top_groups": [
{
"context": "('_eps',)",
"label": "raw_k3",
"methods": 1154,
"unique": 1,
"unique_ratio": 0.001,
"alphabet_size": 0,
"sore": null,
"sore_success": false
},
{
"context": "('get', 'get', 'json')",
"label": "raw_k3",
"methods": 900,
"unique": 14,
"unique_ratio": 0.016,
"alphabet_size": 12,
"sore": null,
"sore_success": false
},
{
"context": "('TestClient', 'TestClient', 'get')",
"label": "raw_k3",
"methods": 179,
"unique": 11,
"unique_ratio": 0.061,
"alphabet_size": 9,
"sore": null,
"sore_success": false
},
{
"context": "('TestClient', 'TestClient', 'post')",
"label": "raw_k3",
"methods": 168,
"unique": 4,
"unique_ratio": 0.024,
"alphabet_size": 5,
"sore": null,
"sore_success": false
},
{
"context": "('post', 'post', 'json')",
"label": "raw_k3",
"methods": 155,
"unique": 6,
"unique_ratio": 0.039,
"alphabet_size": 6,
"sore": null,
"sore_success": false
},
{
"context": "('import_module', 'import_module', 'TestClient')",
"label": "raw_k3",
"methods": 129,
"unique": 2,
"unique_ratio": 0.016,
"alphabet_size": 3,
"sore": null,
"sore_success": false
},
{
"context": "('get', 'get')",
"label": "raw_k3",
"methods": 89,
"unique": 1,
"unique_ratio": 0.011,
"alphabet_size": 1,
"sore": "(get)+",
"sore_success": true
},
{
"context": "('put', 'put', 'json')",
"label": "raw_k3",
"methods": 65,
"unique": 2,
"unique_ratio": 0.031,
"alphabet_size": 3,
"sore": null,
"sore_success": false
},
{
"context": "('openapi', 'openapi', 'get_body_model_name')",
"label": "raw_k3",
"methods": 48,
"unique": 1,
"unique_ratio": 0.021,
"alphabet_size": 2,
"sore": null,
"sore_success": false
},
{
"context": "('openapi', 'openapi', 'snapshot')",
"label": "raw_k3",
"methods": 41,
"unique": 2,
"unique_ratio": 0.049,
"alphabet_size": 3,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('_eps',)",
"packages": 93
},
{
"context": "('get', 'get', 'json')",
"packages": 70
},
{
"context": "('import_module', 'import_module', 'TestClient')",
"packages": 42
},
{
"context": "('post', 'post', 'json')",
"packages": 27
},
{
"context": "('get', 'get')",
"packages": 18
},
{
"context": "('TestClient', 'TestClient', 'get')",
"packages": 15
},
{
"context": "('import_module', 'import_module')",
"packages": 13
},
{
"context": "('put', 'put', 'json')",
"packages": 10
},
{
"context": "('HTTPException', 'HTTPException')",
"packages": 8
},
{
"context": "('len', 'len', 'len')",
"packages": 8
}
]
}
},
"coarsened": {
"seq_count": 4811,
"alphabet_size": 6036,
"alphabet_sample": [
"",
"!=",
"!= M",
"!r",
"\"",
"\" and",
"\"\"",
"\")",
"\")]",
"\", \"\",",
"\"Alread",
"\"Marked",
"\"Notifi",
"\"Path must b",
"\"_{v}\"):",
"\"docs\")",
"\"docs\"):",
"\"en\"",
"\"site",
"\"start",
"\"unk",
"\"{pare",
"\"})",
"#",
"# Ha",
"', '',",
"'/'\"",
"(",
"(\n se",
"(\n self,\n pat"
],
"k1": {
"contexts": 383,
"sore_successes": 10,
"total_methods": 4811,
"methods_in_good": 29,
"coverage": 0.6,
"cross_package_contexts": 59,
"elapsed": 0.005,
"top_groups": [
{
"context": "('RETURN',)",
"label": "coarsened_k1",
"methods": 1274,
"unique": 335,
"unique_ratio": 0.263,
"alphabet_size": 663,
"sore": null,
"sore_success": false
},
{
"context": "('response',)",
"label": "coarsened_k1",
"methods": 1242,
"unique": 199,
"unique_ratio": 0.16,
"alphabet_size": 428,
"sore": null,
"sore_success": false
},
{
"context": "('client',)",
"label": "coarsened_k1",
"methods": 454,
"unique": 119,
"unique_ratio": 0.262,
"alphabet_size": 111,
"sore": null,
"sore_success": false
},
{
"context": "('IF',)",
"label": "coarsened_k1",
"methods": 185,
"unique": 133,
"unique_ratio": 0.719,
"alphabet_size": 1082,
"sore": null,
"sore_success": false
},
{
"context": "('mod',)",
"label": "coarsened_k1",
"methods": 150,
"unique": 14,
"unique_ratio": 0.093,
"alphabet_size": 27,
"sore": null,
"sore_success": false
},
{
"context": "('app',)",
"label": "coarsened_k1",
"methods": 107,
"unique": 57,
"unique_ratio": 0.533,
"alphabet_size": 275,
"sore": null,
"sore_success": false
},
{
"context": "('pytest',)",
"label": "coarsened_k1",
"methods": 66,
"unique": 48,
"unique_ratio": 0.727,
"alphabet_size": 98,
"sore": null,
"sore_success": false
},
{
"context": "('results',)",
"label": "coarsened_k1",
"methods": 58,
"unique": 8,
"unique_ratio": 0.138,
"alphabet_size": 10,
"sore": null,
"sore_success": false
},
{
"context": "('self',)",
"label": "coarsened_k1",
"methods": 52,
"unique": 38,
"unique_ratio": 0.731,
"alphabet_size": 154,
"sore": null,
"sore_success": false
},
{
"context": "('openapi',)",
"label": "coarsened_k1",
"methods": 48,
"unique": 1,
"unique_ratio": 0.021,
"alphabet_size": 5,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('RETURN',)",
"packages": 91
},
{
"context": "('response',)",
"packages": 80
},
{
"context": "('mod',)",
"packages": 49
},
{
"context": "('IF',)",
"packages": 34
},
{
"context": "('client',)",
"packages": 25
},
{
"context": "('LOOP',)",
"packages": 12
},
{
"context": "('result',)",
"packages": 9
},
{
"context": "('self',)",
"packages": 9
},
{
"context": "('data',)",
"packages": 9
},
{
"context": "('EXCEPTION',)",
"packages": 7
}
]
},
"k2": {
"contexts": 888,
"sore_successes": 76,
"total_methods": 4811,
"methods_in_good": 776,
"coverage": 16.1,
"cross_package_contexts": 101,
"elapsed": 0.005,
"top_groups": [
{
"context": "('response', 'client')",
"label": "coarsened_k2",
"methods": 1202,
"unique": 176,
"unique_ratio": 0.146,
"alphabet_size": 101,
"sore": null,
"sore_success": false
},
{
"context": "('client', 'TestClient')",
"label": "coarsened_k2",
"methods": 392,
"unique": 91,
"unique_ratio": 0.232,
"alphabet_size": 88,
"sore": null,
"sore_success": false
},
{
"context": "('RETURN',)",
"label": "coarsened_k2",
"methods": 288,
"unique": 1,
"unique_ratio": 0.003,
"alphabet_size": 1,
"sore": "RETURN",
"sore_success": true
},
{
"context": "('RETURN', 'p')",
"label": "coarsened_k2",
"methods": 156,
"unique": 4,
"unique_ratio": 0.026,
"alphabet_size": 4,
"sore": null,
"sore_success": false
},
{
"context": "('mod', 'importlib')",
"label": "coarsened_k2",
"methods": 143,
"unique": 10,
"unique_ratio": 0.07,
"alphabet_size": 22,
"sore": null,
"sore_success": false
},
{
"context": "('RETURN', 'value')",
"label": "coarsened_k2",
"methods": 66,
"unique": 2,
"unique_ratio": 0.03,
"alphabet_size": 2,
"sore": "RETURN.(value)+",
"sore_success": true
},
{
"context": "('pytest', 'raises')",
"label": "coarsened_k2",
"methods": 61,
"unique": 43,
"unique_ratio": 0.705,
"alphabet_size": 82,
"sore": null,
"sore_success": false
},
{
"context": "('openapi', 'app')",
"label": "coarsened_k2",
"methods": 48,
"unique": 1,
"unique_ratio": 0.021,
"alphabet_size": 5,
"sore": null,
"sore_success": false
},
{
"context": "('dist', 'tmp_path')",
"label": "coarsened_k2",
"methods": 46,
"unique": 41,
"unique_ratio": 0.891,
"alphabet_size": 131,
"sore": null,
"sore_success": false
},
{
"context": "('app', 'FastAPI')",
"label": "coarsened_k2",
"methods": 45,
"unique": 45,
"unique_ratio": 1.0,
"alphabet_size": 251,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('response', 'client')",
"packages": 76
},
{
"context": "('mod', 'importlib')",
"packages": 47
},
{
"context": "('RETURN',)",
"packages": 34
},
{
"context": "('client', 'TestClient')",
"packages": 22
},
{
"context": "('RETURN', 'item')",
"packages": 10
},
{
"context": "('IF', 'not')",
"packages": 10
},
{
"context": "('RETURN', 'username')",
"packages": 7
},
{
"context": "('browser', 'playwright')",
"packages": 7
},
{
"context": "('RETURN', 'p')",
"packages": 7
},
{
"context": "('IF', 'item_id')",
"packages": 6
}
]
},
"k3": {
"contexts": 1047,
"sore_successes": 108,
"total_methods": 4811,
"methods_in_good": 995,
"coverage": 20.7,
"cross_package_contexts": 97,
"elapsed": 0.006,
"top_groups": [
{
"context": "('response', 'client', 'get')",
"label": "coarsened_k3",
"methods": 964,
"unique": 122,
"unique_ratio": 0.127,
"alphabet_size": 82,
"sore": null,
"sore_success": false
},
{
"context": "('client', 'TestClient', 'TestClient')",
"label": "coarsened_k3",
"methods": 392,
"unique": 91,
"unique_ratio": 0.232,
"alphabet_size": 88,
"sore": null,
"sore_success": false
},
{
"context": "('RETURN',)",
"label": "coarsened_k3",
"methods": 288,
"unique": 1,
"unique_ratio": 0.003,
"alphabet_size": 1,
"sore": "RETURN",
"sore_success": true
},
{
"context": "('response', 'client', 'post')",
"label": "coarsened_k3",
"methods": 162,
"unique": 37,
"unique_ratio": 0.228,
"alphabet_size": 42,
"sore": null,
"sore_success": false
},
{
"context": "('mod', 'importlib', 'import_module')",
"label": "coarsened_k3",
"methods": 143,
"unique": 10,
"unique_ratio": 0.07,
"alphabet_size": 22,
"sore": null,
"sore_success": false
},
{
"context": "('RETURN', 'p')",
"label": "coarsened_k3",
"methods": 76,
"unique": 1,
"unique_ratio": 0.013,
"alphabet_size": 2,
"sore": "RETURN.p",
"sore_success": true
},
{
"context": "('RETURN', 'p', 'p')",
"label": "coarsened_k3",
"methods": 72,
"unique": 1,
"unique_ratio": 0.014,
"alphabet_size": 2,
"sore": "RETURN.p+",
"sore_success": true
},
{
"context": "('response', 'client', 'put')",
"label": "coarsened_k3",
"methods": 68,
"unique": 10,
"unique_ratio": 0.147,
"alphabet_size": 12,
"sore": null,
"sore_success": false
},
{
"context": "('RETURN', 'value')",
"label": "coarsened_k3",
"methods": 64,
"unique": 1,
"unique_ratio": 0.016,
"alphabet_size": 2,
"sore": "RETURN.value",
"sore_success": true
},
{
"context": "('pytest', 'raises', 'raises')",
"label": "coarsened_k3",
"methods": 61,
"unique": 43,
"unique_ratio": 0.705,
"alphabet_size": 82,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('response', 'client', 'get')",
"packages": 72
},
{
"context": "('mod', 'importlib', 'import_module')",
"packages": 47
},
{
"context": "('RETURN',)",
"packages": 34
},
{
"context": "('response', 'client', 'post')",
"packages": 28
},
{
"context": "('client', 'TestClient', 'TestClient')",
"packages": 22
},
{
"context": "('response', 'client', 'put')",
"packages": 11
},
{
"context": "('RETURN', 'item')",
"packages": 10
},
{
"context": "('RETURN', 'username')",
"packages": 7
},
{
"context": "('browser', 'playwright', 'chromium')",
"packages": 7
},
{
"context": "('RETURN', 'settings', 'app_name')",
"packages": 6
}
]
}
}
}