tobi/grammar-inference-engine
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions
grammar-inference-engine/experiments/results/flask_reduce_k3_e015.json
tobjend b516b2985d feat: implement Reduce algorithm (Algorithm 4, TODS 2010) 
- bex/reduce.py: Faithful implementation of Reduce with support-weighted
  SOA edit distance, adjunction, iterative merging, and Minimize
- experiments/context_eval.py: Multi-codebase support (RAGSAK + Flask),
  Reduce experiments with thresholds 0.05-0.4
- tests/test_reduce.py: 24 tests covering all Reduce components
- Flask cloned to external_refs/flask for cross-validation

Results:
- RAGSAK: 12.0% coverage (First 3 symbols)
- Flask: 10.7% coverage (First 3 symbols)
- Reduce has minimal impact (1-2 merges per codebase at ε=0.3)
- Coverage ceiling appears to be ~10-12% for prefix-based grouping
2026-07-12 00:11:38 +02:00

712 lines
No EOL
16 KiB
JSON
Raw Blame History

{
"strategy": "Reduce k=3 \u03b5=0.15",
"total_contexts": 598,
"meaningful_contexts": 85,
"total_methods": 1391,
"methods_in_good_groups": 149,
"sore_successes": 21,
"sore_failures": 23,
"skip_reasons": {
"too_large": 2,
"too_diverse": 38,
"large_alphabet": 1
},
"groups": [
{
"context": "('return',)",
"methods": 203,
"unique": 1,
"unique_ratio": 0.005,
"sore": "SKIP(too_large)",
"sore_success": false
},
{
"context": "('@', 'route', 'route')",
"methods": 58,
"unique": 53,
"unique_ratio": 0.914,
"sore": "SKIP(too_large)",
"sore_success": false
},
{
"context": "('return', 'render_template', 'render_template')",
"methods": 33,
"unique": 2,
"unique_ratio": 0.061,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('append', 'append')",
"methods": 33,
"unique": 1,
"unique_ratio": 0.03,
"sore": "(append)+",
"sore_success": true
},
{
"context": "('return', 'isinstance', 'isinstance')",
"methods": 32,
"unique": 2,
"unique_ratio": 0.062,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('Blueprint', 'Blueprint', '@')",
"methods": 21,
"unique": 21,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('raise',)",
"methods": 19,
"unique": 1,
"unique_ratio": 0.053,
"sore": "raise",
"sore_success": true
},
{
"context": "('isinstance', 'isinstance', 'isinstance')",
"methods": 17,
"unique": 6,
"unique_ratio": 0.353,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('append', 'append', 'return')",
"methods": 17,
"unique": 1,
"unique_ratio": 0.059,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('abort', 'abort')",
"methods": 16,
"unique": 1,
"unique_ratio": 0.062,
"sore": "(abort)+",
"sore_success": true
},
{
"context": "('if', 'is', 'return')",
"methods": 10,
"unique": 9,
"unique_ratio": 0.9,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('super', 'super', 'super')",
"methods": 10,
"unique": 10,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('if', 'is not', 'is')",
"methods": 10,
"unique": 9,
"unique_ratio": 0.9,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('return', 'jsonify', 'jsonify')",
"methods": 10,
"unique": 4,
"unique_ratio": 0.4,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('get', 'get', 'if')",
"methods": 9,
"unique": 9,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('if', 'is', 'is not')",
"methods": 9,
"unique": 9,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('pop', 'pop')",
"methods": 9,
"unique": 1,
"unique_ratio": 0.111,
"sore": "(pop)+",
"sore_success": true
},
{
"context": "('setdefault', 'setdefault', 'append')",
"methods": 9,
"unique": 2,
"unique_ratio": 0.222,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('@', 'teardown_request', 'def')",
"methods": 9,
"unique": 8,
"unique_ratio": 0.889,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('Blueprint', 'Blueprint', 'def')",
"methods": 9,
"unique": 7,
"unique_ratio": 0.778,
"sore": "SKIP(large_alphabet)",
"sore_success": false
},
{
"context": "('return', 'str', 'str')",
"methods": 8,
"unique": 3,
"unique_ratio": 0.375,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('eturn', 'lask(', 'lask(')",
"methods": 8,
"unique": 1,
"unique_ratio": 0.125,
"sore": "eturn.(lask()+",
"sore_success": true
},
{
"context": "('Blueprint', 'Blueprint', 'Blueprint')",
"methods": 8,
"unique": 7,
"unique_ratio": 0.875,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('return', 'get', 'get')",
"methods": 7,
"unique": 3,
"unique_ratio": 0.429,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('if', 'isinstance', 'isinstance')",
"methods": 7,
"unique": 7,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('return', 'url_for', 'url_for')",
"methods": 7,
"unique": 1,
"unique_ratio": 0.143,
"sore": "return.(url_for)+",
"sore_success": true
},
{
"context": "('class', 'def', 'get')",
"methods": 7,
"unique": 7,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('if', 'callable', 'callable')",
"methods": 6,
"unique": 6,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('record_once', 'record_once', 'lambda')",
"methods": 6,
"unique": 1,
"unique_ratio": 0.167,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('class', 'def', '__init__')",
"methods": 6,
"unique": 6,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('if', 'get', 'get')",
"methods": 5,
"unique": 5,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('if', 'is', 'raise')",
"methods": 5,
"unique": 5,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('return', '_method_route', '_method_route')",
"methods": 5,
"unique": 1,
"unique_ratio": 0.2,
"sore": "return.(_method_route)+",
"sore_success": true
},
{
"context": "('await', 'sleep', 'sleep')",
"methods": 5,
"unique": 1,
"unique_ratio": 0.2,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('@', 'teardown_appcontext', 'def')",
"methods": 5,
"unique": 5,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('raise', 'ValueError', 'ValueError')",
"methods": 5,
"unique": 1,
"unique_ratio": 0.2,
"sore": "raise.(ValueError)+",
"sore_success": true
},
{
"context": "('cho(', 'cho(')",
"methods": 5,
"unique": 1,
"unique_ratio": 0.2,
"sore": "(cho()+",
"sore_success": true
},
{
"context": "('return', 'dict', 'dict')",
"methods": 5,
"unique": 1,
"unique_ratio": 0.2,
"sore": "return.(dict)+",
"sore_success": true
},
{
"context": "('@', 'errorhandler', 'errorhandler')",
"methods": 5,
"unique": 5,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('class', 'def', 'dispatch_request')",
"methods": 5,
"unique": 3,
"unique_ratio": 0.6,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('@', 'before_request', 'def')",
"methods": 5,
"unique": 5,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('get', 'get', 'get')",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('raise', 'NotImplementedError', 'NotImplementedError')",
"methods": 4,
"unique": 1,
"unique_ratio": 0.25,
"sore": "raise.(NotImplementedError)+",
"sore_success": true
},
{
"context": "('get', 'get', 'return')",
"methods": 4,
"unique": 3,
"unique_ratio": 0.75,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('return', 'tag', 'tag')",
"methods": 4,
"unique": 3,
"unique_ratio": 0.75,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('Flask', 'Flask', '@')",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('test_client', 'test_client', 'get')",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('class', '@', 'errorhandler')",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('return', 'render_template_string', 'render_template_string')",
"methods": 4,
"unique": 1,
"unique_ratio": 0.25,
"sore": "return.(render_template_string)+",
"sore_success": true
},
{
"context": "('@', 'template_filter', 'template_filter')",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('@', 'template_test', 'template_test')",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('raise', 'Forbidden', 'Forbidden')",
"methods": 4,
"unique": 1,
"unique_ratio": 0.25,
"sore": "raise.(Forbidden)+",
"sore_success": true
},
{
"context": "('return', 'Response', 'Response')",
"methods": 4,
"unique": 1,
"unique_ratio": 0.25,
"sore": "return.(Response)+",
"sore_success": true
},
{
"context": "('Flask', 'Flask', 'from_mapping')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('if', 'not in', 'not')",
"methods": 3,
"unique": 2,
"unique_ratio": 0.667,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('return', 'test_client', 'test_client')",
"methods": 3,
"unique": 1,
"unique_ratio": 0.333,
"sore": "return.(test_client)+",
"sore_success": true
},
{
"context": "('yield',)",
"methods": 3,
"unique": 1,
"unique_ratio": 0.333,
"sore": "yield",
"sore_success": true
},
{
"context": "('login', 'login', 'post')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('def', 'wrapper', 'if')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('return', 'join', 'join')",
"methods": 3,
"unique": 2,
"unique_ratio": 0.667,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('return', 'dumps', 'dumps')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('getattr', 'getattr', 'getattr')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('or',)",
"methods": 3,
"unique": 1,
"unique_ratio": 0.333,
"sore": "or",
"sore_success": true
},
{
"context": "('is', 'is not', 'not')",
"methods": 3,
"unique": 2,
"unique_ratio": 0.667,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('raise', 'Exception', 'Exception')",
"methods": 3,
"unique": 1,
"unique_ratio": 0.333,
"sore": "raise.(Exception)+",
"sore_success": true
},
{
"context": "('mkdir', 'mkdir', 'syspath_prepend')",
"methods": 3,
"unique": 2,
"unique_ratio": 0.667,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('lask(', 'lask(', 'dd_url_rule(')",
"methods": 3,
"unique": 2,
"unique_ratio": 0.667,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('em(sys.', 'em(sys.', 's(clic')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('setenv', 'setenv', 'setenv')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('Flask', 'Flask', 'raises')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('yield', 'yield', 'yield')",
"methods": 3,
"unique": 1,
"unique_ratio": 0.333,
"sore": "(yield)+",
"sore_success": true
},
{
"context": "('@', 'stream_with_context', 'def')",
"methods": 3,
"unique": 2,
"unique_ratio": 0.667,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('update', 'update', '@')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('Flask', 'Flask', 'test_client')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('@', 'post', 'post')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('@', 'get', 'get')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('@', 'context_processor', 'def')",
"methods": 3,
"unique": 2,
"unique_ratio": 0.667,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('def', 'my_reverse', 'return')",
"methods": 3,
"unique": 2,
"unique_ratio": 0.667,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('from', 'blueprintapp', 'import')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('from', 'werkzeug', 'routing')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('raise', 'InternalServerError', 'InternalServerError')",
"methods": 3,
"unique": 1,
"unique_ratio": 0.333,
"sore": "raise.(InternalServerError)+",
"sore_success": true
},
{
"context": "('def', 'index', 'return')",
"methods": 3,
"unique": 2,
"unique_ratio": 0.667,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('get_flashed_messages', 'get_flashed_messages', 'list')",
"methods": 3,
"unique": 2,
"unique_ratio": 0.667,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('ndpoin',)",
"methods": 3,
"unique": 1,
"unique_ratio": 0.333,
"sore": "ndpoin",
"sore_success": true
},
{
"context": "('ain_m', 'ain_m', 'p.route(\"/\"')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
}
],
"coverage": 10.7,
"elapsed_seconds": 0.09,
"merge_info": {
"merges": {
"iterations": 1,
"merges": 0,
"comparisons": 11325,
"contexts_before": 598,
"contexts_after": 598,
"threshold": 0.15,
"merge_log": []
},
"threshold": 0.15,
"contexts_before": 598,
"contexts_after": 598
}
}
Reference in a new issue View git blame Copy permalink