tobi/grammar-inference-engine
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions
grammar-inference-engine/experiments/results/coarsen_flask.json

993 lines
24 KiB
JSON
Raw Normal View History

feat: structural coarsening experiment — keeps function names raw, coarsens keywords - coarsen_token() maps tree-sitter captures to categories (RETURN, IF, LOOP, etc.) - Function calls kept as raw text (they ARE the behavioral content) - Only ~5% structural tokens coarsened - Results: Flask coverage 17.5% -> 28.8% (+11.3%), RAGSAK unchanged (95% calls) - Cross-package shapes emerge: ('IF', 'KW', 'RETURN') in 4 Flask packages
2026-07-12 00:56:09 +02:00
{
"name": "Flask",
"raw": {
"seq_count": 1391,
"alphabet_size": 1055,
"alphabet_sample": [
"",
"!r",
"\" * 100)",
"\")",
"\", \"index\",",
"\", de",
"\", su",
"\"/\")",
"\"/t",
"\"X-Foo\"",
"\"about\"",
"\"ba",
"\"cus",
"\"fake a",
"\"fo",
"\"foo.vi",
"\"h",
"\"hello\"",
"\"lat",
"\"mer",
"\"nes",
"\"someth",
"\"spam\")",
"\"static",
"# ip addre",
"(\"\"), 40",
"(\"\", hea",
"()",
"(test",
"({\"msg\":"
],
"k1": {
"contexts": 373,
"sore_successes": 22,
"total_methods": 1391,
"methods_in_good": 64,
"coverage": 4.6,
"cross_package_contexts": 58,
"elapsed": 0.0,
"top_groups": [
{
"context": "('_eps',)",
"label": "raw_k1",
"methods": 282,
"unique": 1,
"unique_ratio": 0.004,
"alphabet_size": 0,
"sore": null,
"sore_success": false
},
{
"context": "('isinstance',)",
"label": "raw_k1",
"methods": 66,
"unique": 20,
"unique_ratio": 0.303,
"alphabet_size": 55,
"sore": null,
"sore_success": false
},
{
"context": "('route',)",
"label": "raw_k1",
"methods": 59,
"unique": 54,
"unique_ratio": 0.915,
"alphabet_size": 148,
"sore": null,
"sore_success": false
},
{
"context": "('append',)",
"label": "raw_k1",
"methods": 53,
"unique": 3,
"unique_ratio": 0.057,
"alphabet_size": 4,
"sore": null,
"sore_success": false
},
{
"context": "('get',)",
"label": "raw_k1",
"methods": 50,
"unique": 40,
"unique_ratio": 0.8,
"alphabet_size": 97,
"sore": null,
"sore_success": false
},
{
"context": "('Blueprint',)",
"label": "raw_k1",
"methods": 41,
"unique": 37,
"unique_ratio": 0.902,
"alphabet_size": 121,
"sore": null,
"sore_success": false
},
{
"context": "('render_template',)",
"label": "raw_k1",
"methods": 33,
"unique": 2,
"unique_ratio": 0.061,
"alphabet_size": 2,
"sore": null,
"sore_success": false
},
{
"context": "('Flask',)",
"label": "raw_k1",
"methods": 28,
"unique": 26,
"unique_ratio": 0.929,
"alphabet_size": 63,
"sore": null,
"sore_success": false
},
{
"context": "('abort',)",
"label": "raw_k1",
"methods": 17,
"unique": 2,
"unique_ratio": 0.118,
"alphabet_size": 3,
"sore": null,
"sore_success": false
},
{
"context": "('super',)",
"label": "raw_k1",
"methods": 16,
"unique": 12,
"unique_ratio": 0.75,
"alphabet_size": 33,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('_eps',)",
"packages": 9
},
{
"context": "('get',)",
"packages": 7
},
{
"context": "('render_template',)",
"packages": 6
},
{
"context": "('Flask',)",
"packages": 4
},
{
"context": "('pop',)",
"packages": 4
},
{
"context": "('isinstance',)",
"packages": 4
},
{
"context": "('setdefault',)",
"packages": 4
},
{
"context": "('post',)",
"packages": 3
},
{
"context": "('app_context',)",
"packages": 3
},
{
"context": "('test_client',)",
"packages": 3
}
]
},
"k2": {
"contexts": 389,
"sore_successes": 22,
"total_methods": 1391,
"methods_in_good": 64,
"coverage": 4.6,
"cross_package_contexts": 54,
"elapsed": 0.001,
"top_groups": [
{
"context": "('_eps',)",
"label": "raw_k2",
"methods": 282,
"unique": 1,
"unique_ratio": 0.004,
"alphabet_size": 0,
"sore": null,
"sore_success": false
},
{
"context": "('isinstance', 'isinstance')",
"label": "raw_k2",
"methods": 66,
"unique": 20,
"unique_ratio": 0.303,
"alphabet_size": 55,
"sore": null,
"sore_success": false
},
{
"context": "('route', 'route')",
"label": "raw_k2",
"methods": 59,
"unique": 54,
"unique_ratio": 0.915,
"alphabet_size": 148,
"sore": null,
"sore_success": false
},
{
"context": "('append', 'append')",
"label": "raw_k2",
"methods": 53,
"unique": 3,
"unique_ratio": 0.057,
"alphabet_size": 4,
"sore": null,
"sore_success": false
},
{
"context": "('get', 'get')",
"label": "raw_k2",
"methods": 50,
"unique": 40,
"unique_ratio": 0.8,
"alphabet_size": 97,
"sore": null,
"sore_success": false
},
{
"context": "('Blueprint', 'Blueprint')",
"label": "raw_k2",
"methods": 41,
"unique": 37,
"unique_ratio": 0.902,
"alphabet_size": 121,
"sore": null,
"sore_success": false
},
{
"context": "('render_template', 'render_template')",
"label": "raw_k2",
"methods": 33,
"unique": 2,
"unique_ratio": 0.061,
"alphabet_size": 2,
"sore": null,
"sore_success": false
},
{
"context": "('Flask', 'Flask')",
"label": "raw_k2",
"methods": 28,
"unique": 26,
"unique_ratio": 0.929,
"alphabet_size": 63,
"sore": null,
"sore_success": false
},
{
"context": "('abort', 'abort')",
"label": "raw_k2",
"methods": 17,
"unique": 2,
"unique_ratio": 0.118,
"alphabet_size": 3,
"sore": null,
"sore_success": false
},
{
"context": "('super', 'super')",
"label": "raw_k2",
"methods": 16,
"unique": 12,
"unique_ratio": 0.75,
"alphabet_size": 33,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('_eps',)",
"packages": 9
},
{
"context": "('get', 'get')",
"packages": 7
},
{
"context": "('render_template', 'render_template')",
"packages": 6
},
{
"context": "('Flask', 'Flask')",
"packages": 4
},
{
"context": "('pop', 'pop')",
"packages": 4
},
{
"context": "('isinstance', 'isinstance')",
"packages": 4
},
{
"context": "('setdefault', 'setdefault')",
"packages": 4
},
{
"context": "('post', 'post')",
"packages": 3
},
{
"context": "('app_context', 'app_context')",
"packages": 3
},
{
"context": "('test_client', 'test_client')",
"packages": 3
}
]
},
"k3": {
"contexts": 599,
"sore_successes": 43,
"total_methods": 1391,
"methods_in_good": 243,
"coverage": 17.5,
"cross_package_contexts": 40,
"elapsed": 0.001,
"top_groups": [
{
"context": "('_eps',)",
"label": "raw_k3",
"methods": 282,
"unique": 1,
"unique_ratio": 0.004,
"alphabet_size": 0,
"sore": null,
"sore_success": false
},
{
"context": "('isinstance', 'isinstance', 'isinstance')",
"label": "raw_k3",
"methods": 66,
"unique": 20,
"unique_ratio": 0.303,
"alphabet_size": 55,
"sore": null,
"sore_success": false
},
{
"context": "('append', 'append')",
"label": "raw_k3",
"methods": 51,
"unique": 1,
"unique_ratio": 0.02,
"alphabet_size": 1,
"sore": "(append)+",
"sore_success": true
},
{
"context": "('route', 'route', 'index')",
"label": "raw_k3",
"methods": 42,
"unique": 37,
"unique_ratio": 0.881,
"alphabet_size": 73,
"sore": null,
"sore_success": false
},
{
"context": "('render_template', 'render_template')",
"label": "raw_k3",
"methods": 31,
"unique": 1,
"unique_ratio": 0.032,
"alphabet_size": 1,
"sore": "(render_template)+",
"sore_success": true
},
{
"context": "('super', 'super', 'super')",
"label": "raw_k3",
"methods": 16,
"unique": 12,
"unique_ratio": 0.75,
"alphabet_size": 33,
"sore": null,
"sore_success": false
},
{
"context": "('abort', 'abort')",
"label": "raw_k3",
"methods": 16,
"unique": 1,
"unique_ratio": 0.062,
"alphabet_size": 1,
"sore": "(abort)+",
"sore_success": true
},
{
"context": "('get', 'get')",
"label": "raw_k3",
"methods": 10,
"unique": 1,
"unique_ratio": 0.1,
"alphabet_size": 1,
"sore": "(get)+",
"sore_success": true
},
{
"context": "('pop', 'pop')",
"label": "raw_k3",
"methods": 10,
"unique": 1,
"unique_ratio": 0.1,
"alphabet_size": 1,
"sore": "(pop)+",
"sore_success": true
},
{
"context": "('setdefault', 'setdefault', 'append')",
"label": "raw_k3",
"methods": 9,
"unique": 1,
"unique_ratio": 0.111,
"alphabet_size": 2,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('_eps',)",
"packages": 9
},
{
"context": "('render_template', 'render_template')",
"packages": 6
},
{
"context": "('get', 'get', 'get')",
"packages": 4
},
{
"context": "('isinstance', 'isinstance', 'isinstance')",
"packages": 4
},
{
"context": "('Flask', 'Flask', 'from_mapping')",
"packages": 3
},
{
"context": "('test_client', 'test_client')",
"packages": 3
},
{
"context": "('get', 'get')",
"packages": 3
},
{
"context": "('super', 'super', 'super')",
"packages": 3
},
{
"context": "('type', 'type', 'type')",
"packages": 3
},
{
"context": "('getattr', 'getattr', 'getattr')",
"packages": 3
}
]
}
},
"coarsened": {
"seq_count": 1391,
"alphabet_size": 1063,
"alphabet_sample": [
"",
"!r",
"\" * 100)",
"\")",
"\", \"index\",",
"\", de",
"\", su",
"\"/\")",
"\"/t",
"\"X-Foo\"",
"\"about\"",
"\"ba",
"\"cus",
"\"fake a",
"\"fo",
"\"foo.vi",
"\"h",
"\"hello\"",
"\"lat",
"\"mer",
"\"nes",
"\"someth",
"\"spam\")",
"\"static",
"# ip addre",
"(\"\"), 40",
"(\"\", hea",
"()",
"(test",
"({\"msg\":"
],
"k1": {
"contexts": 166,
"sore_successes": 3,
"total_methods": 1391,
"methods_in_good": 20,
"coverage": 1.4,
"cross_package_contexts": 22,
"elapsed": 0.0,
"top_groups": [
{
"context": "('RETURN',)",
"label": "coarsened_k1",
"methods": 448,
"unique": 101,
"unique_ratio": 0.225,
"alphabet_size": 97,
"sore": null,
"sore_success": false
},
{
"context": "('ATTR',)",
"label": "coarsened_k1",
"methods": 120,
"unique": 112,
"unique_ratio": 0.933,
"alphabet_size": 296,
"sore": null,
"sore_success": false
},
{
"context": "('IF',)",
"label": "coarsened_k1",
"methods": 117,
"unique": 109,
"unique_ratio": 0.932,
"alphabet_size": 209,
"sore": null,
"sore_success": false
},
{
"context": "('KW',)",
"label": "coarsened_k1",
"methods": 91,
"unique": 80,
"unique_ratio": 0.879,
"alphabet_size": 246,
"sore": null,
"sore_success": false
},
{
"context": "('EXCEPTION',)",
"label": "coarsened_k1",
"methods": 78,
"unique": 43,
"unique_ratio": 0.551,
"alphabet_size": 83,
"sore": null,
"sore_success": false
},
{
"context": "('append',)",
"label": "coarsened_k1",
"methods": 51,
"unique": 3,
"unique_ratio": 0.059,
"alphabet_size": 4,
"sore": null,
"sore_success": false
},
{
"context": "('FUN',)",
"label": "coarsened_k1",
"methods": 48,
"unique": 46,
"unique_ratio": 0.958,
"alphabet_size": 127,
"sore": null,
"sore_success": false
},
{
"context": "('Blueprint',)",
"label": "coarsened_k1",
"methods": 40,
"unique": 36,
"unique_ratio": 0.9,
"alphabet_size": 119,
"sore": null,
"sore_success": false
},
{
"context": "('get',)",
"label": "coarsened_k1",
"methods": 25,
"unique": 24,
"unique_ratio": 0.96,
"alphabet_size": 61,
"sore": null,
"sore_success": false
},
{
"context": "('Flask',)",
"label": "coarsened_k1",
"methods": 23,
"unique": 23,
"unique_ratio": 1.0,
"alphabet_size": 69,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('RETURN',)",
"packages": 14
},
{
"context": "('get',)",
"packages": 7
},
{
"context": "('LOOP',)",
"packages": 6
},
{
"context": "('IF',)",
"packages": 6
},
{
"context": "('FUN',)",
"packages": 5
},
{
"context": "('KW',)",
"packages": 5
},
{
"context": "('pop',)",
"packages": 4
},
{
"context": "('EXCEPTION',)",
"packages": 4
},
{
"context": "('setdefault',)",
"packages": 4
},
{
"context": "('Flask',)",
"packages": 3
}
]
},
"k2": {
"contexts": 353,
"sore_successes": 23,
"total_methods": 1391,
"methods_in_good": 346,
"coverage": 24.9,
"cross_package_contexts": 49,
"elapsed": 0.0,
"top_groups": [
{
"context": "('RETURN',)",
"label": "coarsened_k2",
"methods": 234,
"unique": 1,
"unique_ratio": 0.004,
"alphabet_size": 1,
"sore": "RETURN",
"sore_success": true
},
{
"context": "('ATTR', 'ATTR')",
"label": "coarsened_k2",
"methods": 80,
"unique": 74,
"unique_ratio": 0.925,
"alphabet_size": 171,
"sore": null,
"sore_success": false
},
{
"context": "('IF', 'KW')",
"label": "coarsened_k2",
"methods": 67,
"unique": 60,
"unique_ratio": 0.896,
"alphabet_size": 134,
"sore": null,
"sore_success": false
},
{
"context": "('append', 'append')",
"label": "coarsened_k2",
"methods": 51,
"unique": 3,
"unique_ratio": 0.059,
"alphabet_size": 4,
"sore": null,
"sore_success": false
},
{
"context": "('Blueprint', 'Blueprint')",
"label": "coarsened_k2",
"methods": 40,
"unique": 36,
"unique_ratio": 0.9,
"alphabet_size": 119,
"sore": null,
"sore_success": false
},
{
"context": "('RETURN', 'render_template')",
"label": "coarsened_k2",
"methods": 33,
"unique": 2,
"unique_ratio": 0.061,
"alphabet_size": 3,
"sore": null,
"sore_success": false
},
{
"context": "('KW', 'FUN')",
"label": "coarsened_k2",
"methods": 32,
"unique": 30,
"unique_ratio": 0.938,
"alphabet_size": 122,
"sore": null,
"sore_success": false
},
{
"context": "('RETURN', 'isinstance')",
"label": "coarsened_k2",
"methods": 32,
"unique": 2,
"unique_ratio": 0.062,
"alphabet_size": 6,
"sore": null,
"sore_success": false
},
{
"context": "('get', 'get')",
"label": "coarsened_k2",
"methods": 25,
"unique": 24,
"unique_ratio": 0.96,
"alphabet_size": 61,
"sore": null,
"sore_success": false
},
{
"context": "('ATTR', 'route')",
"label": "coarsened_k2",
"methods": 24,
"unique": 22,
"unique_ratio": 0.917,
"alphabet_size": 103,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('RETURN',)",
"packages": 9
},
{
"context": "('get', 'get')",
"packages": 7
},
{
"context": "('RETURN', 'render_template')",
"packages": 6
},
{
"context": "('LOOP', 'KW')",
"packages": 6
},
{
"context": "('IF', 'KW')",
"packages": 5
},
{
"context": "('KW', 'FUN')",
"packages": 4
},
{
"context": "('pop', 'pop')",
"packages": 4
},
{
"context": "('setdefault', 'setdefault')",
"packages": 4
},
{
"context": "('IF', 'isinstance')",
"packages": 4
},
{
"context": "('Flask', 'Flask')",
"packages": 3
}
]
},
"k3": {
"contexts": 529,
"sore_successes": 30,
"total_methods": 1391,
"methods_in_good": 400,
"coverage": 28.8,
"cross_package_contexts": 47,
"elapsed": 0.0,
"top_groups": [
{
"context": "('RETURN',)",
"label": "coarsened_k3",
"methods": 234,
"unique": 1,
"unique_ratio": 0.004,
"alphabet_size": 1,
"sore": "RETURN",
"sore_success": true
},
{
"context": "('IF', 'KW', 'KW')",
"label": "coarsened_k3",
"methods": 34,
"unique": 29,
"unique_ratio": 0.853,
"alphabet_size": 73,
"sore": null,
"sore_success": false
},
{
"context": "('ATTR', 'ATTR', 'route')",
"label": "coarsened_k3",
"methods": 34,
"unique": 31,
"unique_ratio": 0.912,
"alphabet_size": 74,
"sore": null,
"sore_success": false
},
{
"context": "('RETURN', 'render_template', 'render_template')",
"label": "coarsened_k3",
"methods": 33,
"unique": 2,
"unique_ratio": 0.061,
"alphabet_size": 3,
"sore": null,
"sore_success": false
},
{
"context": "('append', 'append')",
"label": "coarsened_k3",
"methods": 33,
"unique": 1,
"unique_ratio": 0.03,
"alphabet_size": 1,
"sore": "(append)+",
"sore_success": true
},
{
"context": "('RETURN', 'isinstance', 'isinstance')",
"label": "coarsened_k3",
"methods": 32,
"unique": 2,
"unique_ratio": 0.062,
"alphabet_size": 6,
"sore": null,
"sore_success": false
},
{
"context": "('ATTR', 'ATTR', 'FUN')",
"label": "coarsened_k3",
"methods": 31,
"unique": 28,
"unique_ratio": 0.903,
"alphabet_size": 92,
"sore": null,
"sore_success": false
},
{
"context": "('ATTR', 'route', 'route')",
"label": "coarsened_k3",
"methods": 24,
"unique": 22,
"unique_ratio": 0.917,
"alphabet_size": 103,
"sore": null,
"sore_success": false
},
{
"context": "('EXCEPTION',)",
"label": "coarsened_k3",
"methods": 21,
"unique": 1,
"unique_ratio": 0.048,
"alphabet_size": 1,
"sore": "EXCEPTION",
"sore_success": true
},
{
"context": "('Blueprint', 'Blueprint', 'ATTR')",
"label": "coarsened_k3",
"methods": 21,
"unique": 20,
"unique_ratio": 0.952,
"alphabet_size": 76,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('RETURN',)",
"packages": 9
},
{
"context": "('RETURN', 'render_template', 'render_template')",
"packages": 6
},
{
"context": "('get', 'get', 'get')",
"packages": 4
},
{
"context": "('IF', 'KW', 'RETURN')",
"packages": 4
},
{
"context": "('get', 'get', 'IF')",
"packages": 4
},
{
"context": "('IF', 'KW', 'KW')",
"packages": 4
},
{
"context": "('IF', 'isinstance', 'isinstance')",
"packages": 4
},
{
"context": "('Flask', 'Flask', 'from_mapping')",
"packages": 3
},
{
"context": "('RETURN', 'test_client', 'test_client')",
"packages": 3
},
{
"context": "('RETURN', 'get', 'get')",
"packages": 3
}
]
}
}
}
Reference in a new issue Copy permalink