tobi/grammar-inference-engine
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions
grammar-inference-engine/experiments/results/flask_reduce_k1_e02.json

456 lines
9.5 KiB
JSON
Raw Normal View History

feat: implement Reduce algorithm (Algorithm 4, TODS 2010) - bex/reduce.py: Faithful implementation of Reduce with support-weighted SOA edit distance, adjunction, iterative merging, and Minimize - experiments/context_eval.py: Multi-codebase support (RAGSAK + Flask), Reduce experiments with thresholds 0.05-0.4 - tests/test_reduce.py: 24 tests covering all Reduce components - Flask cloned to external_refs/flask for cross-validation Results: - RAGSAK: 12.0% coverage (First 3 symbols) - Flask: 10.7% coverage (First 3 symbols) - Reduce has minimal impact (1-2 merges per codebase at ε=0.3) - Coverage ceiling appears to be ~10-12% for prefix-based grouping
2026-07-12 00:11:38 +02:00
{
"strategy": "Reduce k=1 \u03b5=0.2",
"total_contexts": 228,
"meaningful_contexts": 53,
"total_methods": 1391,
"methods_in_good_groups": 19,
"sore_successes": 2,
"sore_failures": 14,
"skip_reasons": {
"too_large": 5,
"too_diverse": 29,
"large_alphabet": 3
},
"groups": [
{
"context": "('return',)",
"methods": 393,
"unique": 88,
"unique_ratio": 0.224,
"sore": "SKIP(too_large)",
"sore_success": false
},
{
"context": "('if',)",
"methods": 116,
"unique": 109,
"unique_ratio": 0.94,
"sore": "SKIP(too_large)",
"sore_success": false
},
{
"context": "('@',)",
"methods": 110,
"unique": 102,
"unique_ratio": 0.927,
"sore": "SKIP(too_large)",
"sore_success": false
},
{
"context": "('raise',)",
"methods": 60,
"unique": 27,
"unique_ratio": 0.45,
"sore": "SKIP(too_large)",
"sore_success": false
},
{
"context": "('append',)",
"methods": 51,
"unique": 3,
"unique_ratio": 0.059,
"sore": "SKIP(too_large)",
"sore_success": false
},
{
"context": "('class',)",
"methods": 47,
"unique": 45,
"unique_ratio": 0.957,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('Blueprint',)",
"methods": 40,
"unique": 37,
"unique_ratio": 0.925,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('def',)",
"methods": 38,
"unique": 36,
"unique_ratio": 0.947,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('get',)",
"methods": 25,
"unique": 24,
"unique_ratio": 0.96,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('Flask',)",
"methods": 23,
"unique": 23,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('isinstance',)",
"methods": 17,
"unique": 6,
"unique_ratio": 0.353,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('abort',)",
"methods": 16,
"unique": 1,
"unique_ratio": 0.062,
"sore": "(abort)+",
"sore_success": true
},
{
"context": "('setdefault',)",
"methods": 14,
"unique": 6,
"unique_ratio": 0.429,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('pop',)",
"methods": 13,
"unique": 5,
"unique_ratio": 0.385,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('try',)",
"methods": 13,
"unique": 13,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('for',)",
"methods": 12,
"unique": 12,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('from',)",
"methods": 12,
"unique": 12,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('eturn',)",
"methods": 11,
"unique": 4,
"unique_ratio": 0.364,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('super',)",
"methods": 10,
"unique": 10,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('yield',)",
"methods": 9,
"unique": 5,
"unique_ratio": 0.556,
"sore": "SKIP(large_alphabet)",
"sore_success": false
},
{
"context": "('raises',)",
"methods": 8,
"unique": 7,
"unique_ratio": 0.875,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('name_',)",
"methods": 8,
"unique": 6,
"unique_ratio": 0.75,
"sore": "SKIP(large_alphabet)",
"sore_success": false
},
{
"context": "('login',)",
"methods": 7,
"unique": 6,
"unique_ratio": 0.857,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('app_context',)",
"methods": 6,
"unique": 6,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('_get_current_object',)",
"methods": 6,
"unique": 6,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('record_once',)",
"methods": 6,
"unique": 1,
"unique_ratio": 0.167,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('cho(',)",
"methods": 6,
"unique": 2,
"unique_ratio": 0.333,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('not',)",
"methods": 5,
"unique": 5,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('or',)",
"methods": 5,
"unique": 3,
"unique_ratio": 0.6,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('await',)",
"methods": 5,
"unique": 1,
"unique_ratio": 0.2,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('test_client',)",
"methods": 5,
"unique": 5,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('mkdir',)",
"methods": 5,
"unique": 4,
"unique_ratio": 0.8,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('ef',)",
"methods": 5,
"unique": 5,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('lask(',)",
"methods": 5,
"unique": 4,
"unique_ratio": 0.8,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('get_flashed_messages',)",
"methods": 5,
"unique": 4,
"unique_ratio": 0.8,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('import',)",
"methods": 4,
"unique": 3,
"unique_ratio": 0.75,
"sore": "SKIP(large_alphabet)",
"sore_success": false
},
{
"context": "('join',)",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('is',)",
"methods": 4,
"unique": 3,
"unique_ratio": 0.75,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('write_text',)",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('em(sys.',)",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('setenv',)",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('ain_m',)",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('post',)",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('get_db',)",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('getLogger',)",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('getattr',)",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('setattr',)",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('rom',)",
"methods": 3,
"unique": 2,
"unique_ratio": 0.667,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('nvoke(',)",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('update',)",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('t',)",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('lask.u',)",
"methods": 3,
"unique": 2,
"unique_ratio": 0.667,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('ndpoin',)",
"methods": 3,
"unique": 1,
"unique_ratio": 0.333,
"sore": "ndpoin",
"sore_success": true
}
],
"coverage": 1.4,
"elapsed_seconds": 0.07,
"merge_info": {
"merges": {
"iterations": 1,
"merges": 0,
"comparisons": 4095,
"contexts_before": 228,
"contexts_after": 228,
"threshold": 0.2,
"merge_log": []
},
"threshold": 0.2,
"contexts_before": 228,
"contexts_after": 228
}
}
Reference in a new issue Copy permalink