tobi/grammar-inference-engine
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions
grammar-inference-engine/experiments/results/flask_first_k_sym_2.json
tobjend b516b2985d feat: implement Reduce algorithm (Algorithm 4, TODS 2010) 
- bex/reduce.py: Faithful implementation of Reduce with support-weighted
  SOA edit distance, adjunction, iterative merging, and Minimize
- experiments/context_eval.py: Multi-codebase support (RAGSAK + Flask),
  Reduce experiments with thresholds 0.05-0.4
- tests/test_reduce.py: 24 tests covering all Reduce components
- Flask cloned to external_refs/flask for cross-validation

Results:
- RAGSAK: 12.0% coverage (First 3 symbols)
- Flask: 10.7% coverage (First 3 symbols)
- Reduce has minimal impact (1-2 merges per codebase at ε=0.3)
- Coverage ceiling appears to be ~10-12% for prefix-based grouping
2026-07-12 00:11:38 +02:00

786 lines
No EOL
17 KiB
JSON
Raw Blame History

{
"strategy": "Option B: First 2 symbols",
"total_contexts": 430,
"meaningful_contexts": 96,
"total_methods": 1391,
"methods_in_good_groups": 102,
"sore_successes": 18,
"sore_failures": 23,
"skip_reasons": {
"too_large": 3,
"too_diverse": 50,
"large_alphabet": 2
},
"groups": [
{
"context": "('return',)",
"methods": 203,
"unique": 1,
"unique_ratio": 0.005,
"sore": "SKIP(too_large)",
"sore_success": false
},
{
"context": "('@', 'route')",
"methods": 58,
"unique": 53,
"unique_ratio": 0.914,
"sore": "SKIP(too_large)",
"sore_success": false
},
{
"context": "('append', 'append')",
"methods": 51,
"unique": 3,
"unique_ratio": 0.059,
"sore": "SKIP(too_large)",
"sore_success": false
},
{
"context": "('Blueprint', 'Blueprint')",
"methods": 40,
"unique": 37,
"unique_ratio": 0.925,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('return', 'render_template')",
"methods": 33,
"unique": 2,
"unique_ratio": 0.061,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('if', 'is')",
"methods": 33,
"unique": 30,
"unique_ratio": 0.909,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('return', 'isinstance')",
"methods": 32,
"unique": 2,
"unique_ratio": 0.062,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('class', 'def')",
"methods": 30,
"unique": 28,
"unique_ratio": 0.933,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('get', 'get')",
"methods": 25,
"unique": 24,
"unique_ratio": 0.96,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('Flask', 'Flask')",
"methods": 23,
"unique": 23,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('raise',)",
"methods": 19,
"unique": 1,
"unique_ratio": 0.053,
"sore": "raise",
"sore_success": true
},
{
"context": "('isinstance', 'isinstance')",
"methods": 17,
"unique": 6,
"unique_ratio": 0.353,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('abort', 'abort')",
"methods": 16,
"unique": 1,
"unique_ratio": 0.062,
"sore": "(abort)+",
"sore_success": true
},
{
"context": "('setdefault', 'setdefault')",
"methods": 14,
"unique": 6,
"unique_ratio": 0.429,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('pop', 'pop')",
"methods": 13,
"unique": 5,
"unique_ratio": 0.385,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('if', 'not')",
"methods": 13,
"unique": 12,
"unique_ratio": 0.923,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('for', 'in')",
"methods": 12,
"unique": 12,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('class', 'class')",
"methods": 11,
"unique": 11,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('super', 'super')",
"methods": 10,
"unique": 10,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('if', 'is not')",
"methods": 10,
"unique": 9,
"unique_ratio": 0.9,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('if', 'return')",
"methods": 10,
"unique": 10,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('return', 'jsonify')",
"methods": 10,
"unique": 4,
"unique_ratio": 0.4,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('@', 'teardown_request')",
"methods": 9,
"unique": 8,
"unique_ratio": 0.889,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('return', 'str')",
"methods": 8,
"unique": 3,
"unique_ratio": 0.375,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('raises', 'raises')",
"methods": 8,
"unique": 7,
"unique_ratio": 0.875,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('eturn', 'lask(')",
"methods": 8,
"unique": 1,
"unique_ratio": 0.125,
"sore": "eturn.(lask()+",
"sore_success": true
},
{
"context": "('name_', 'name_')",
"methods": 8,
"unique": 6,
"unique_ratio": 0.75,
"sore": "SKIP(large_alphabet)",
"sore_success": false
},
{
"context": "('return', 'get')",
"methods": 7,
"unique": 3,
"unique_ratio": 0.429,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('login', 'login')",
"methods": 7,
"unique": 6,
"unique_ratio": 0.857,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('if', 'isinstance')",
"methods": 7,
"unique": 7,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('return', 'url_for')",
"methods": 7,
"unique": 1,
"unique_ratio": 0.143,
"sore": "return.(url_for)+",
"sore_success": true
},
{
"context": "('app_context', 'app_context')",
"methods": 6,
"unique": 6,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('_get_current_object', '_get_current_object')",
"methods": 6,
"unique": 6,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('if', 'callable')",
"methods": 6,
"unique": 6,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('record_once', 'record_once')",
"methods": 6,
"unique": 1,
"unique_ratio": 0.167,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('cho(', 'cho(')",
"methods": 6,
"unique": 2,
"unique_ratio": 0.333,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('if', 'get')",
"methods": 5,
"unique": 5,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('if', 'raise')",
"methods": 5,
"unique": 4,
"unique_ratio": 0.8,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('return', '_method_route')",
"methods": 5,
"unique": 1,
"unique_ratio": 0.2,
"sore": "return.(_method_route)+",
"sore_success": true
},
{
"context": "('await', 'sleep')",
"methods": 5,
"unique": 1,
"unique_ratio": 0.2,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('test_client', 'test_client')",
"methods": 5,
"unique": 5,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('@', 'teardown_appcontext')",
"methods": 5,
"unique": 5,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('raise', 'ValueError')",
"methods": 5,
"unique": 1,
"unique_ratio": 0.2,
"sore": "raise.(ValueError)+",
"sore_success": true
},
{
"context": "('mkdir', 'mkdir')",
"methods": 5,
"unique": 4,
"unique_ratio": 0.8,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('lask(', 'lask(')",
"methods": 5,
"unique": 4,
"unique_ratio": 0.8,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('return', 'dict')",
"methods": 5,
"unique": 1,
"unique_ratio": 0.2,
"sore": "return.(dict)+",
"sore_success": true
},
{
"context": "('@', 'errorhandler')",
"methods": 5,
"unique": 5,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('get_flashed_messages', 'get_flashed_messages')",
"methods": 5,
"unique": 4,
"unique_ratio": 0.8,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('@', 'before_request')",
"methods": 5,
"unique": 5,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('join', 'join')",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('if', 'and')",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('try', 'import')",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('raise', 'NotImplementedError')",
"methods": 4,
"unique": 1,
"unique_ratio": 0.25,
"sore": "raise.(NotImplementedError)+",
"sore_success": true
},
{
"context": "('return', 'tag')",
"methods": 4,
"unique": 3,
"unique_ratio": 0.75,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('def', 'decorator')",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('write_text', 'write_text')",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('em(sys.', 'em(sys.')",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('setenv', 'setenv')",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('class', '@')",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('return', 'render_template_string')",
"methods": 4,
"unique": 1,
"unique_ratio": 0.25,
"sore": "return.(render_template_string)+",
"sore_success": true
},
{
"context": "('@', 'template_filter')",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('@', 'template_test')",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('raise', 'Forbidden')",
"methods": 4,
"unique": 1,
"unique_ratio": 0.25,
"sore": "raise.(Forbidden)+",
"sore_success": true
},
{
"context": "('return', 'Response')",
"methods": 4,
"unique": 1,
"unique_ratio": 0.25,
"sore": "return.(Response)+",
"sore_success": true
},
{
"context": "('ain_m', 'ain_m')",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('post', 'post')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('get_db', 'get_db')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('if', 'not in')",
"methods": 3,
"unique": 2,
"unique_ratio": 0.667,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('return', 'test_client')",
"methods": 3,
"unique": 1,
"unique_ratio": 0.333,
"sore": "return.(test_client)+",
"sore_success": true
},
{
"context": "('yield',)",
"methods": 3,
"unique": 1,
"unique_ratio": 0.333,
"sore": "yield",
"sore_success": true
},
{
"context": "('try', 'return')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('import', 'warnings')",
"methods": 3,
"unique": 2,
"unique_ratio": 0.667,
"sore": "SKIP(large_alphabet)",
"sore_success": false
},
{
"context": "('def', 'wrapper')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('return', 'join')",
"methods": 3,
"unique": 2,
"unique_ratio": 0.667,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('getLogger', 'getLogger')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('return', 'dumps')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('getattr', 'getattr')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('if', 'in')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('or',)",
"methods": 3,
"unique": 1,
"unique_ratio": 0.333,
"sore": "or",
"sore_success": true
},
{
"context": "('is', 'is not')",
"methods": 3,
"unique": 2,
"unique_ratio": 0.667,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('raise', 'Exception')",
"methods": 3,
"unique": 1,
"unique_ratio": 0.333,
"sore": "raise.(Exception)+",
"sore_success": true
},
{
"context": "('setattr', 'setattr')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('ef', 'reate_app(')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('nvoke(', 'nvoke(')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('yield', 'yield')",
"methods": 3,
"unique": 1,
"unique_ratio": 0.333,
"sore": "(yield)+",
"sore_success": true
},
{
"context": "('@', 'stream_with_context')",
"methods": 3,
"unique": 2,
"unique_ratio": 0.667,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('update', 'update')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('@', 'post')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('@', 'get')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('@', 'context_processor')",
"methods": 3,
"unique": 2,
"unique_ratio": 0.667,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('def', 'my_reverse')",
"methods": 3,
"unique": 2,
"unique_ratio": 0.667,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('from', 'blueprintapp')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('from', 'werkzeug')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('raise', 'InternalServerError')",
"methods": 3,
"unique": 1,
"unique_ratio": 0.333,
"sore": "raise.(InternalServerError)+",
"sore_success": true
},
{
"context": "('def', 'index')",
"methods": 3,
"unique": 2,
"unique_ratio": 0.667,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('ndpoin',)",
"methods": 3,
"unique": 1,
"unique_ratio": 0.333,
"sore": "ndpoin",
"sore_success": true
}
],
"coverage": 7.3,
"elapsed_seconds": 0.08
}
Reference in a new issue View git blame Copy permalink