tobi/grammar-inference-engine
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions
grammar-inference-engine/experiments/results/flask_two_d_p1_s2.json

786 lines
18 KiB
JSON
Raw Normal View History

feat: implement Reduce algorithm (Algorithm 4, TODS 2010) - bex/reduce.py: Faithful implementation of Reduce with support-weighted SOA edit distance, adjunction, iterative merging, and Minimize - experiments/context_eval.py: Multi-codebase support (RAGSAK + Flask), Reduce experiments with thresholds 0.05-0.4 - tests/test_reduce.py: 24 tests covering all Reduce components - Flask cloned to external_refs/flask for cross-validation Results: - RAGSAK: 12.0% coverage (First 3 symbols) - Flask: 10.7% coverage (First 3 symbols) - Reduce has minimal impact (1-2 merges per codebase at ε=0.3) - Coverage ceiling appears to be ~10-12% for prefix-based grouping
2026-07-12 00:11:38 +02:00
{
"strategy": "Option C: Path k=1 + Symbol k=2",
"total_contexts": 512,
"meaningful_contexts": 96,
"total_methods": 1391,
"methods_in_good_groups": 141,
"sore_successes": 22,
"sore_failures": 22,
"skip_reasons": {
"too_large": 2,
"too_diverse": 48,
"large_alphabet": 2
},
"groups": [
{
"context": "('tests', 'return')",
"methods": 177,
"unique": 1,
"unique_ratio": 0.006,
"sore": "SKIP(too_large)",
"sore_success": false
},
{
"context": "('tests', '@', 'route')",
"methods": 58,
"unique": 53,
"unique_ratio": 0.914,
"sore": "SKIP(too_large)",
"sore_success": false
},
{
"context": "('tests', 'append', 'append')",
"methods": 45,
"unique": 3,
"unique_ratio": 0.067,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('tests', 'Blueprint', 'Blueprint')",
"methods": 40,
"unique": 37,
"unique_ratio": 0.925,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'class', 'def')",
"methods": 28,
"unique": 26,
"unique_ratio": 0.929,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'return', 'render_template')",
"methods": 25,
"unique": 2,
"unique_ratio": 0.08,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('tests', 'return', 'isinstance')",
"methods": 24,
"unique": 1,
"unique_ratio": 0.042,
"sore": "return.(isinstance)+",
"sore_success": true
},
{
"context": "('flask', 'if', 'is')",
"methods": 22,
"unique": 20,
"unique_ratio": 0.909,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'Flask', 'Flask')",
"methods": 21,
"unique": 21,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'abort', 'abort')",
"methods": 16,
"unique": 1,
"unique_ratio": 0.062,
"sore": "(abort)+",
"sore_success": true
},
{
"context": "('tests', 'isinstance', 'isinstance')",
"methods": 16,
"unique": 5,
"unique_ratio": 0.312,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('flask', 'if', 'not')",
"methods": 13,
"unique": 12,
"unique_ratio": 0.923,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('type_check', 'return')",
"methods": 13,
"unique": 1,
"unique_ratio": 0.077,
"sore": "return",
"sore_success": true
},
{
"context": "('tests', 'raise')",
"methods": 12,
"unique": 1,
"unique_ratio": 0.083,
"sore": "raise",
"sore_success": true
},
{
"context": "('tests', 'get', 'get')",
"methods": 11,
"unique": 10,
"unique_ratio": 0.909,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'class', 'class')",
"methods": 11,
"unique": 11,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('flask', 'get', 'get')",
"methods": 9,
"unique": 9,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('sansio', 'if', 'is')",
"methods": 9,
"unique": 9,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'return', 'jsonify')",
"methods": 9,
"unique": 4,
"unique_ratio": 0.444,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('tests', '@', 'teardown_request')",
"methods": 9,
"unique": 8,
"unique_ratio": 0.889,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'setdefault', 'setdefault')",
"methods": 9,
"unique": 3,
"unique_ratio": 0.333,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('tests', 'pop', 'pop')",
"methods": 9,
"unique": 2,
"unique_ratio": 0.222,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('tests', 'raises', 'raises')",
"methods": 8,
"unique": 7,
"unique_ratio": 0.875,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'eturn', 'lask(')",
"methods": 8,
"unique": 1,
"unique_ratio": 0.125,
"sore": "eturn.(lask()+",
"sore_success": true
},
{
"context": "('tests', 'name_', 'name_')",
"methods": 8,
"unique": 6,
"unique_ratio": 0.75,
"sore": "SKIP(large_alphabet)",
"sore_success": false
},
{
"context": "('tests', 'login', 'login')",
"methods": 7,
"unique": 6,
"unique_ratio": 0.857,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('flask', 'return')",
"methods": 7,
"unique": 1,
"unique_ratio": 0.143,
"sore": "return",
"sore_success": true
},
{
"context": "('flask', 'super', 'super')",
"methods": 7,
"unique": 7,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('flask', 'if', 'is not')",
"methods": 7,
"unique": 6,
"unique_ratio": 0.857,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('json', 'return', 'isinstance')",
"methods": 7,
"unique": 2,
"unique_ratio": 0.286,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('tests', 'return', 'str')",
"methods": 7,
"unique": 2,
"unique_ratio": 0.286,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('flask', 'for', 'in')",
"methods": 6,
"unique": 6,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('flask', 'if', 'return')",
"methods": 6,
"unique": 6,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('flask', '_get_current_object', '_get_current_object')",
"methods": 6,
"unique": 6,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('sansio', 'if', 'callable')",
"methods": 6,
"unique": 6,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('sansio', 'append', 'append')",
"methods": 6,
"unique": 2,
"unique_ratio": 0.333,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('sansio', 'record_once', 'record_once')",
"methods": 6,
"unique": 1,
"unique_ratio": 0.167,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('tests', 'cho(', 'cho(')",
"methods": 6,
"unique": 2,
"unique_ratio": 0.333,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('tests', 'return', 'url_for')",
"methods": 6,
"unique": 1,
"unique_ratio": 0.167,
"sore": "return.(url_for)+",
"sore_success": true
},
{
"context": "('tests', 'app_context', 'app_context')",
"methods": 5,
"unique": 5,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('flask', 'if', 'get')",
"methods": 5,
"unique": 5,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('json', 'raise')",
"methods": 5,
"unique": 1,
"unique_ratio": 0.2,
"sore": "raise",
"sore_success": true
},
{
"context": "('sansio', 'return', '_method_route')",
"methods": 5,
"unique": 1,
"unique_ratio": 0.2,
"sore": "return.(_method_route)+",
"sore_success": true
},
{
"context": "('tests', 'await', 'sleep')",
"methods": 5,
"unique": 1,
"unique_ratio": 0.2,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('tests', 'test_client', 'test_client')",
"methods": 5,
"unique": 5,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', '@', 'teardown_appcontext')",
"methods": 5,
"unique": 5,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'raise', 'ValueError')",
"methods": 5,
"unique": 1,
"unique_ratio": 0.2,
"sore": "raise.(ValueError)+",
"sore_success": true
},
{
"context": "('tests', 'mkdir', 'mkdir')",
"methods": 5,
"unique": 4,
"unique_ratio": 0.8,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('tests', 'lask(', 'lask(')",
"methods": 5,
"unique": 4,
"unique_ratio": 0.8,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('tests', 'return', 'dict')",
"methods": 5,
"unique": 1,
"unique_ratio": 0.2,
"sore": "return.(dict)+",
"sore_success": true
},
{
"context": "('tests', '@', 'errorhandler')",
"methods": 5,
"unique": 5,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'get_flashed_messages', 'get_flashed_messages')",
"methods": 5,
"unique": 4,
"unique_ratio": 0.8,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('tests', '@', 'before_request')",
"methods": 5,
"unique": 5,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('flask', 'return', 'get')",
"methods": 4,
"unique": 3,
"unique_ratio": 0.75,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('sansio', 'def', 'decorator')",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'write_text', 'write_text')",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'em(sys.', 'em(sys.')",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'setenv', 'setenv')",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'class', '@')",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'return', 'render_template_string')",
"methods": 4,
"unique": 1,
"unique_ratio": 0.25,
"sore": "return.(render_template_string)+",
"sore_success": true
},
{
"context": "('tests', '@', 'template_filter')",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', '@', 'template_test')",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'raise', 'Forbidden')",
"methods": 4,
"unique": 1,
"unique_ratio": 0.25,
"sore": "raise.(Forbidden)+",
"sore_success": true
},
{
"context": "('tests', 'ain_m', 'ain_m')",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'post', 'post')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('flaskr', 'get_db', 'get_db')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'return', 'test_client')",
"methods": 3,
"unique": 1,
"unique_ratio": 0.333,
"sore": "return.(test_client)+",
"sore_success": true
},
{
"context": "('tests', 'return', 'get')",
"methods": 3,
"unique": 1,
"unique_ratio": 0.333,
"sore": "return.(get)+",
"sore_success": true
},
{
"context": "('tests', 'yield')",
"methods": 3,
"unique": 1,
"unique_ratio": 0.333,
"sore": "yield",
"sore_success": true
},
{
"context": "('flask', 'try', 'return')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('flask', 'import', 'warnings')",
"methods": 3,
"unique": 2,
"unique_ratio": 0.667,
"sore": "SKIP(large_alphabet)",
"sore_success": false
},
{
"context": "('flask', 'join', 'join')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('flask', 'if', 'isinstance')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('flask', 'try', 'import')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('flask', 'raise', 'NotImplementedError')",
"methods": 3,
"unique": 1,
"unique_ratio": 0.333,
"sore": "raise.(NotImplementedError)+",
"sore_success": true
},
{
"context": "('json', 'if', 'return')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('json', 'return', 'tag')",
"methods": 3,
"unique": 2,
"unique_ratio": 0.667,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('sansio', 'or')",
"methods": 3,
"unique": 1,
"unique_ratio": 0.333,
"sore": "or",
"sore_success": true
},
{
"context": "('sansio', 'if', 'is not')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('sansio', 'setdefault', 'setdefault')",
"methods": 3,
"unique": 1,
"unique_ratio": 0.333,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('tests', 'raise', 'Exception')",
"methods": 3,
"unique": 1,
"unique_ratio": 0.333,
"sore": "raise.(Exception)+",
"sore_success": true
},
{
"context": "('tests', 'setattr', 'setattr')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'ef', 'reate_app(')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'nvoke(', 'nvoke(')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'yield', 'yield')",
"methods": 3,
"unique": 1,
"unique_ratio": 0.333,
"sore": "(yield)+",
"sore_success": true
},
{
"context": "('tests', '@', 'stream_with_context')",
"methods": 3,
"unique": 2,
"unique_ratio": 0.667,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('tests', 'update', 'update')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', '@', 'post')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', '@', 'get')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', '@', 'context_processor')",
"methods": 3,
"unique": 2,
"unique_ratio": 0.667,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('tests', 'def', 'my_reverse')",
"methods": 3,
"unique": 2,
"unique_ratio": 0.667,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('tests', 'from', 'blueprintapp')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'from', 'werkzeug')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'raise', 'InternalServerError')",
"methods": 3,
"unique": 1,
"unique_ratio": 0.333,
"sore": "raise.(InternalServerError)+",
"sore_success": true
},
{
"context": "('tests', 'def', 'index')",
"methods": 3,
"unique": 2,
"unique_ratio": 0.667,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('tests', 'ndpoin')",
"methods": 3,
"unique": 1,
"unique_ratio": 0.333,
"sore": "ndpoin",
"sore_success": true
}
],
"coverage": 10.1,
"elapsed_seconds": 0.07
}
Reference in a new issue Copy permalink