grammar-inference-engine/experiments/results/coarsen_flask.json
tobjend dd183f6241 feat: 4-codebase evaluation — conventions vs completions tradeoff
Added kotlinx.coroutines (1040 .kt files) and FastAPI (1129 .py files).

Key findings across 4 codebases:
- Coarsening trades per-package coverage for cross-package reach
- FastAPI: coverage IMPROVES (12.4% -> 20.7%), cross-pkg +15
- Coroutines: cross-pkg +15, real exception-handling patterns
- RAGSAK: cross-pkg +18, null-check conventions across 8 packages
- Flask: slight cross-pkg loss, rendering conventions still visible

The conventions vs completions tradeoff is real and measurable.
2026-07-12 01:14:09 +02:00

993 lines
No EOL
24 KiB
JSON

{
"name": "Flask",
"raw": {
"seq_count": 1424,
"alphabet_size": 1055,
"alphabet_sample": [
"",
"!r",
"\" * 100)",
"\")",
"\", \"index\",",
"\", de",
"\", su",
"\"/\")",
"\"/t",
"\"X-Foo\"",
"\"about\"",
"\"ba",
"\"cus",
"\"fake a",
"\"fo",
"\"foo.vi",
"\"h",
"\"hello\"",
"\"lat",
"\"mer",
"\"nes",
"\"someth",
"\"spam\")",
"\"static",
"# ip addre",
"(\"\"), 40",
"(\"\", hea",
"()",
"(test",
"({\"msg\":"
],
"k1": {
"contexts": 373,
"sore_successes": 22,
"total_methods": 1424,
"methods_in_good": 64,
"coverage": 4.5,
"cross_package_contexts": 58,
"elapsed": 0.001,
"top_groups": [
{
"context": "('_eps',)",
"label": "raw_k1",
"methods": 315,
"unique": 1,
"unique_ratio": 0.003,
"alphabet_size": 0,
"sore": null,
"sore_success": false
},
{
"context": "('isinstance',)",
"label": "raw_k1",
"methods": 66,
"unique": 20,
"unique_ratio": 0.303,
"alphabet_size": 55,
"sore": null,
"sore_success": false
},
{
"context": "('route',)",
"label": "raw_k1",
"methods": 59,
"unique": 54,
"unique_ratio": 0.915,
"alphabet_size": 148,
"sore": null,
"sore_success": false
},
{
"context": "('append',)",
"label": "raw_k1",
"methods": 53,
"unique": 3,
"unique_ratio": 0.057,
"alphabet_size": 4,
"sore": null,
"sore_success": false
},
{
"context": "('get',)",
"label": "raw_k1",
"methods": 50,
"unique": 40,
"unique_ratio": 0.8,
"alphabet_size": 97,
"sore": null,
"sore_success": false
},
{
"context": "('Blueprint',)",
"label": "raw_k1",
"methods": 41,
"unique": 37,
"unique_ratio": 0.902,
"alphabet_size": 121,
"sore": null,
"sore_success": false
},
{
"context": "('render_template',)",
"label": "raw_k1",
"methods": 33,
"unique": 2,
"unique_ratio": 0.061,
"alphabet_size": 2,
"sore": null,
"sore_success": false
},
{
"context": "('Flask',)",
"label": "raw_k1",
"methods": 28,
"unique": 26,
"unique_ratio": 0.929,
"alphabet_size": 63,
"sore": null,
"sore_success": false
},
{
"context": "('abort',)",
"label": "raw_k1",
"methods": 17,
"unique": 2,
"unique_ratio": 0.118,
"alphabet_size": 3,
"sore": null,
"sore_success": false
},
{
"context": "('setdefault',)",
"label": "raw_k1",
"methods": 16,
"unique": 5,
"unique_ratio": 0.312,
"alphabet_size": 6,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('_eps',)",
"packages": 10
},
{
"context": "('get',)",
"packages": 7
},
{
"context": "('render_template',)",
"packages": 6
},
{
"context": "('Flask',)",
"packages": 4
},
{
"context": "('pop',)",
"packages": 4
},
{
"context": "('isinstance',)",
"packages": 4
},
{
"context": "('setdefault',)",
"packages": 4
},
{
"context": "('test_client',)",
"packages": 3
},
{
"context": "('post',)",
"packages": 3
},
{
"context": "('app_context',)",
"packages": 3
}
]
},
"k2": {
"contexts": 389,
"sore_successes": 22,
"total_methods": 1424,
"methods_in_good": 64,
"coverage": 4.5,
"cross_package_contexts": 54,
"elapsed": 0.002,
"top_groups": [
{
"context": "('_eps',)",
"label": "raw_k2",
"methods": 315,
"unique": 1,
"unique_ratio": 0.003,
"alphabet_size": 0,
"sore": null,
"sore_success": false
},
{
"context": "('isinstance', 'isinstance')",
"label": "raw_k2",
"methods": 66,
"unique": 20,
"unique_ratio": 0.303,
"alphabet_size": 55,
"sore": null,
"sore_success": false
},
{
"context": "('route', 'route')",
"label": "raw_k2",
"methods": 59,
"unique": 54,
"unique_ratio": 0.915,
"alphabet_size": 148,
"sore": null,
"sore_success": false
},
{
"context": "('append', 'append')",
"label": "raw_k2",
"methods": 53,
"unique": 3,
"unique_ratio": 0.057,
"alphabet_size": 4,
"sore": null,
"sore_success": false
},
{
"context": "('get', 'get')",
"label": "raw_k2",
"methods": 50,
"unique": 40,
"unique_ratio": 0.8,
"alphabet_size": 97,
"sore": null,
"sore_success": false
},
{
"context": "('Blueprint', 'Blueprint')",
"label": "raw_k2",
"methods": 41,
"unique": 37,
"unique_ratio": 0.902,
"alphabet_size": 121,
"sore": null,
"sore_success": false
},
{
"context": "('render_template', 'render_template')",
"label": "raw_k2",
"methods": 33,
"unique": 2,
"unique_ratio": 0.061,
"alphabet_size": 2,
"sore": null,
"sore_success": false
},
{
"context": "('Flask', 'Flask')",
"label": "raw_k2",
"methods": 28,
"unique": 26,
"unique_ratio": 0.929,
"alphabet_size": 63,
"sore": null,
"sore_success": false
},
{
"context": "('abort', 'abort')",
"label": "raw_k2",
"methods": 17,
"unique": 2,
"unique_ratio": 0.118,
"alphabet_size": 3,
"sore": null,
"sore_success": false
},
{
"context": "('setdefault', 'setdefault')",
"label": "raw_k2",
"methods": 16,
"unique": 5,
"unique_ratio": 0.312,
"alphabet_size": 6,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('_eps',)",
"packages": 10
},
{
"context": "('get', 'get')",
"packages": 7
},
{
"context": "('render_template', 'render_template')",
"packages": 6
},
{
"context": "('Flask', 'Flask')",
"packages": 4
},
{
"context": "('pop', 'pop')",
"packages": 4
},
{
"context": "('isinstance', 'isinstance')",
"packages": 4
},
{
"context": "('setdefault', 'setdefault')",
"packages": 4
},
{
"context": "('test_client', 'test_client')",
"packages": 3
},
{
"context": "('post', 'post')",
"packages": 3
},
{
"context": "('app_context', 'app_context')",
"packages": 3
}
]
},
"k3": {
"contexts": 599,
"sore_successes": 43,
"total_methods": 1424,
"methods_in_good": 243,
"coverage": 17.1,
"cross_package_contexts": 40,
"elapsed": 0.001,
"top_groups": [
{
"context": "('_eps',)",
"label": "raw_k3",
"methods": 315,
"unique": 1,
"unique_ratio": 0.003,
"alphabet_size": 0,
"sore": null,
"sore_success": false
},
{
"context": "('isinstance', 'isinstance', 'isinstance')",
"label": "raw_k3",
"methods": 66,
"unique": 20,
"unique_ratio": 0.303,
"alphabet_size": 55,
"sore": null,
"sore_success": false
},
{
"context": "('append', 'append')",
"label": "raw_k3",
"methods": 51,
"unique": 1,
"unique_ratio": 0.02,
"alphabet_size": 1,
"sore": "(append)+",
"sore_success": true
},
{
"context": "('route', 'route', 'index')",
"label": "raw_k3",
"methods": 42,
"unique": 37,
"unique_ratio": 0.881,
"alphabet_size": 73,
"sore": null,
"sore_success": false
},
{
"context": "('render_template', 'render_template')",
"label": "raw_k3",
"methods": 31,
"unique": 1,
"unique_ratio": 0.032,
"alphabet_size": 1,
"sore": "(render_template)+",
"sore_success": true
},
{
"context": "('super', 'super', 'super')",
"label": "raw_k3",
"methods": 16,
"unique": 12,
"unique_ratio": 0.75,
"alphabet_size": 33,
"sore": null,
"sore_success": false
},
{
"context": "('abort', 'abort')",
"label": "raw_k3",
"methods": 16,
"unique": 1,
"unique_ratio": 0.062,
"alphabet_size": 1,
"sore": "(abort)+",
"sore_success": true
},
{
"context": "('get', 'get')",
"label": "raw_k3",
"methods": 10,
"unique": 1,
"unique_ratio": 0.1,
"alphabet_size": 1,
"sore": "(get)+",
"sore_success": true
},
{
"context": "('pop', 'pop')",
"label": "raw_k3",
"methods": 10,
"unique": 1,
"unique_ratio": 0.1,
"alphabet_size": 1,
"sore": "(pop)+",
"sore_success": true
},
{
"context": "('setdefault', 'setdefault', 'append')",
"label": "raw_k3",
"methods": 9,
"unique": 1,
"unique_ratio": 0.111,
"alphabet_size": 2,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('_eps',)",
"packages": 10
},
{
"context": "('render_template', 'render_template')",
"packages": 6
},
{
"context": "('get', 'get', 'get')",
"packages": 4
},
{
"context": "('isinstance', 'isinstance', 'isinstance')",
"packages": 4
},
{
"context": "('Flask', 'Flask', 'from_mapping')",
"packages": 3
},
{
"context": "('test_client', 'test_client')",
"packages": 3
},
{
"context": "('get', 'get')",
"packages": 3
},
{
"context": "('type', 'type', 'type')",
"packages": 3
},
{
"context": "('super', 'super', 'super')",
"packages": 3
},
{
"context": "('getattr', 'getattr', 'getattr')",
"packages": 3
}
]
}
},
"coarsened": {
"seq_count": 1424,
"alphabet_size": 2569,
"alphabet_sample": [
"",
"!r",
"\"",
"\" * 100)",
"\")",
"\",",
"\", \"index\",",
"\", 301)\n\n wi",
"\", de",
"\", su",
"\"/\")",
"\"/t",
"\"Conten",
"\"X-Bar\"",
"\"X-Foo\"",
"\"]\n wea",
"\"about\"",
"\"ba",
"\"cus",
"\"def",
"\"f",
"\"fake a",
"\"fo",
"\"foo.vi",
"\"h",
"\"hello\"",
"\"in",
"\"lat",
"\"mer",
"\"nes"
],
"k1": {
"contexts": 205,
"sore_successes": 2,
"total_methods": 1424,
"methods_in_good": 4,
"coverage": 0.3,
"cross_package_contexts": 27,
"elapsed": 0.001,
"top_groups": [
{
"context": "('RETURN',)",
"label": "coarsened_k1",
"methods": 420,
"unique": 170,
"unique_ratio": 0.405,
"alphabet_size": 243,
"sore": null,
"sore_success": false
},
{
"context": "('IF',)",
"label": "coarsened_k1",
"methods": 94,
"unique": 91,
"unique_ratio": 0.968,
"alphabet_size": 447,
"sore": null,
"sore_success": false
},
{
"context": "('app',)",
"label": "coarsened_k1",
"methods": 79,
"unique": 79,
"unique_ratio": 1.0,
"alphabet_size": 353,
"sore": null,
"sore_success": false
},
{
"context": "('EXCEPTION',)",
"label": "coarsened_k1",
"methods": 74,
"unique": 49,
"unique_ratio": 0.662,
"alphabet_size": 190,
"sore": null,
"sore_success": false
},
{
"context": "('@',)",
"label": "coarsened_k1",
"methods": 69,
"unique": 68,
"unique_ratio": 0.986,
"alphabet_size": 336,
"sore": null,
"sore_success": false
},
{
"context": "('self',)",
"label": "coarsened_k1",
"methods": 65,
"unique": 65,
"unique_ratio": 1.0,
"alphabet_size": 209,
"sore": null,
"sore_success": false
},
{
"context": "('flask',)",
"label": "coarsened_k1",
"methods": 62,
"unique": 26,
"unique_ratio": 0.419,
"alphabet_size": 59,
"sore": null,
"sore_success": false
},
{
"context": "('class',)",
"label": "coarsened_k1",
"methods": 42,
"unique": 42,
"unique_ratio": 1.0,
"alphabet_size": 300,
"sore": null,
"sore_success": false
},
{
"context": "('def',)",
"label": "coarsened_k1",
"methods": 32,
"unique": 31,
"unique_ratio": 0.969,
"alphabet_size": 169,
"sore": null,
"sore_success": false
},
{
"context": "('bp',)",
"label": "coarsened_k1",
"methods": 31,
"unique": 29,
"unique_ratio": 0.935,
"alphabet_size": 134,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('RETURN',)",
"packages": 14
},
{
"context": "('app',)",
"packages": 7
},
{
"context": "('LOOP',)",
"packages": 6
},
{
"context": "('self',)",
"packages": 6
},
{
"context": "('IF',)",
"packages": 5
},
{
"context": "('def',)",
"packages": 5
},
{
"context": "('response',)",
"packages": 4
},
{
"context": "('EXCEPTION',)",
"packages": 4
},
{
"context": "('@',)",
"packages": 3
},
{
"context": "('class',)",
"packages": 3
}
]
},
"k2": {
"contexts": 570,
"sore_successes": 18,
"total_methods": 1424,
"methods_in_good": 225,
"coverage": 15.8,
"cross_package_contexts": 34,
"elapsed": 0.001,
"top_groups": [
{
"context": "('RETURN',)",
"label": "coarsened_k2",
"methods": 127,
"unique": 1,
"unique_ratio": 0.008,
"alphabet_size": 1,
"sore": "RETURN",
"sore_success": true
},
{
"context": "('RETURN', 'flask')",
"label": "coarsened_k2",
"methods": 79,
"unique": 30,
"unique_ratio": 0.38,
"alphabet_size": 38,
"sore": null,
"sore_success": false
},
{
"context": "('@', 'app')",
"label": "coarsened_k2",
"methods": 65,
"unique": 64,
"unique_ratio": 0.985,
"alphabet_size": 319,
"sore": null,
"sore_success": false
},
{
"context": "('RETURN', 'self')",
"label": "coarsened_k2",
"methods": 32,
"unique": 27,
"unique_ratio": 0.844,
"alphabet_size": 56,
"sore": null,
"sore_success": false
},
{
"context": "('RETURN', 'isinstance')",
"label": "coarsened_k2",
"methods": 32,
"unique": 10,
"unique_ratio": 0.312,
"alphabet_size": 21,
"sore": null,
"sore_success": false
},
{
"context": "('bp', 'flask')",
"label": "coarsened_k2",
"methods": 31,
"unique": 29,
"unique_ratio": 0.935,
"alphabet_size": 134,
"sore": null,
"sore_success": false
},
{
"context": "('RETURN', 's')",
"label": "coarsened_k2",
"methods": 23,
"unique": 1,
"unique_ratio": 0.043,
"alphabet_size": 2,
"sore": "RETURN.s",
"sore_success": true
},
{
"context": "('IF', 'self')",
"label": "coarsened_k2",
"methods": 22,
"unique": 22,
"unique_ratio": 1.0,
"alphabet_size": 145,
"sore": null,
"sore_success": false
},
{
"context": "('flask', 'g')",
"label": "coarsened_k2",
"methods": 20,
"unique": 8,
"unique_ratio": 0.4,
"alphabet_size": 29,
"sore": null,
"sore_success": false
},
{
"context": "('flask', 'session')",
"label": "coarsened_k2",
"methods": 20,
"unique": 11,
"unique_ratio": 0.55,
"alphabet_size": 30,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('RETURN', 'render_template')",
"packages": 5
},
{
"context": "('RETURN', 'self')",
"packages": 5
},
{
"context": "('RETURN',)",
"packages": 4
},
{
"context": "('RETURN', 'app')",
"packages": 4
},
{
"context": "('app', 'Flask')",
"packages": 3
},
{
"context": "('app', 'app_context')",
"packages": 3
},
{
"context": "('RETURN', 'isinstance')",
"packages": 3
},
{
"context": "('EXCEPTION', 'NotImplementedError')",
"packages": 3
},
{
"context": "('self', 'app')",
"packages": 3
},
{
"context": "('IF', 'isinstance')",
"packages": 3
}
]
},
"k3": {
"contexts": 699,
"sore_successes": 23,
"total_methods": 1424,
"methods_in_good": 247,
"coverage": 17.3,
"cross_package_contexts": 27,
"elapsed": 0.001,
"top_groups": [
{
"context": "('RETURN',)",
"label": "coarsened_k3",
"methods": 127,
"unique": 1,
"unique_ratio": 0.008,
"alphabet_size": 1,
"sore": "RETURN",
"sore_success": true
},
{
"context": "('@', 'app', 'route')",
"label": "coarsened_k3",
"methods": 44,
"unique": 44,
"unique_ratio": 1.0,
"alphabet_size": 257,
"sore": null,
"sore_success": false
},
{
"context": "('RETURN', 'isinstance', 'isinstance')",
"label": "coarsened_k3",
"methods": 32,
"unique": 10,
"unique_ratio": 0.312,
"alphabet_size": 21,
"sore": null,
"sore_success": false
},
{
"context": "('bp', 'flask', 'Blueprint')",
"label": "coarsened_k3",
"methods": 31,
"unique": 29,
"unique_ratio": 0.935,
"alphabet_size": 134,
"sore": null,
"sore_success": false
},
{
"context": "('RETURN', 'flask', 'request')",
"label": "coarsened_k3",
"methods": 25,
"unique": 8,
"unique_ratio": 0.32,
"alphabet_size": 12,
"sore": null,
"sore_success": false
},
{
"context": "('RETURN', 'flask', 'render_template')",
"label": "coarsened_k3",
"methods": 25,
"unique": 4,
"unique_ratio": 0.16,
"alphabet_size": 8,
"sore": null,
"sore_success": false
},
{
"context": "('RETURN', 's')",
"label": "coarsened_k3",
"methods": 23,
"unique": 1,
"unique_ratio": 0.043,
"alphabet_size": 2,
"sore": "RETURN.s",
"sore_success": true
},
{
"context": "('app', 'flask', 'Flask')",
"label": "coarsened_k3",
"methods": 18,
"unique": 18,
"unique_ratio": 1.0,
"alphabet_size": 111,
"sore": null,
"sore_success": false
},
{
"context": "('flask', 'abort', 'abort')",
"label": "coarsened_k3",
"methods": 16,
"unique": 1,
"unique_ratio": 0.062,
"alphabet_size": 2,
"sore": "flask.(abort)+",
"sore_success": true
},
{
"context": "('isinstance', 'isinstance', 'isinstance')",
"label": "coarsened_k3",
"methods": 16,
"unique": 13,
"unique_ratio": 0.812,
"alphabet_size": 26,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('RETURN', 'render_template', 'render_template')",
"packages": 5
},
{
"context": "('RETURN',)",
"packages": 4
},
{
"context": "('app', 'Flask', 'Flask')",
"packages": 3
},
{
"context": "('RETURN', 'app', 'test_client')",
"packages": 3
},
{
"context": "('app', 'app_context', 'app_context')",
"packages": 3
},
{
"context": "('RETURN', 'isinstance', 'isinstance')",
"packages": 3
},
{
"context": "('EXCEPTION', 'NotImplementedError', 'NotImplementedError')",
"packages": 3
},
{
"context": "('self', 'app', 'app')",
"packages": 3
},
{
"context": "('IF', 'isinstance', 'isinstance')",
"packages": 3
},
{
"context": "('kwargs', 'setdefault', 'setdefault')",
"packages": 3
}
]
}
}
}