tobi/grammar-inference-engine
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions
grammar-inference-engine/experiments/results/flask_two_d_p2_s1.json
tobjend b516b2985d feat: implement Reduce algorithm (Algorithm 4, TODS 2010) 
- bex/reduce.py: Faithful implementation of Reduce with support-weighted
  SOA edit distance, adjunction, iterative merging, and Minimize
- experiments/context_eval.py: Multi-codebase support (RAGSAK + Flask),
  Reduce experiments with thresholds 0.05-0.4
- tests/test_reduce.py: 24 tests covering all Reduce components
- Flask cloned to external_refs/flask for cross-validation

Results:
- RAGSAK: 12.0% coverage (First 3 symbols)
- Flask: 10.7% coverage (First 3 symbols)
- Reduce has minimal impact (1-2 merges per codebase at ε=0.3)
- Coverage ceiling appears to be ~10-12% for prefix-based grouping
2026-07-12 00:11:38 +02:00

570 lines
No EOL
13 KiB
JSON
Raw Blame History

{
"strategy": "Option C: Path k=2 + Symbol k=1",
"total_contexts": 297,
"meaningful_contexts": 69,
"total_methods": 1391,
"methods_in_good_groups": 30,
"sore_successes": 5,
"sore_failures": 19,
"skip_reasons": {
"too_large": 3,
"large_alphabet": 5,
"too_diverse": 37
},
"groups": [
{
"context": "('tests', 'return')",
"methods": 281,
"unique": 33,
"unique_ratio": 0.117,
"sore": "SKIP(too_large)",
"sore_success": false
},
{
"context": "('tests', '@')",
"methods": 108,
"unique": 100,
"unique_ratio": 0.926,
"sore": "SKIP(too_large)",
"sore_success": false
},
{
"context": "('src', 'flask', 'if')",
"methods": 68,
"unique": 63,
"unique_ratio": 0.926,
"sore": "SKIP(too_large)",
"sore_success": false
},
{
"context": "('tests', 'raise')",
"methods": 47,
"unique": 24,
"unique_ratio": 0.511,
"sore": "SKIP(large_alphabet)",
"sore_success": false
},
{
"context": "('tests', 'append')",
"methods": 45,
"unique": 3,
"unique_ratio": 0.067,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('tests', 'class')",
"methods": 44,
"unique": 42,
"unique_ratio": 0.955,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'Blueprint')",
"methods": 40,
"unique": 37,
"unique_ratio": 0.925,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('src', 'flask', 'return')",
"methods": 32,
"unique": 25,
"unique_ratio": 0.781,
"sore": "SKIP(large_alphabet)",
"sore_success": false
},
{
"context": "('flask', 'sansio', 'if')",
"methods": 28,
"unique": 27,
"unique_ratio": 0.964,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('flask', 'json', 'return')",
"methods": 26,
"unique": 20,
"unique_ratio": 0.769,
"sore": "SKIP(large_alphabet)",
"sore_success": false
},
{
"context": "('tests', 'def')",
"methods": 22,
"unique": 20,
"unique_ratio": 0.909,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'Flask')",
"methods": 21,
"unique": 21,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'type_check', 'return')",
"methods": 21,
"unique": 7,
"unique_ratio": 0.333,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('flask', 'sansio', 'return')",
"methods": 16,
"unique": 11,
"unique_ratio": 0.688,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('tests', 'abort')",
"methods": 16,
"unique": 1,
"unique_ratio": 0.062,
"sore": "(abort)+",
"sore_success": true
},
{
"context": "('tests', 'isinstance')",
"methods": 16,
"unique": 5,
"unique_ratio": 0.312,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('src', 'flask', 'try')",
"methods": 12,
"unique": 12,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('flask', 'sansio', 'def')",
"methods": 11,
"unique": 11,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'eturn')",
"methods": 11,
"unique": 4,
"unique_ratio": 0.364,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('src', 'flask', 'get')",
"methods": 9,
"unique": 9,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'setdefault')",
"methods": 9,
"unique": 3,
"unique_ratio": 0.333,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('tests', 'from')",
"methods": 9,
"unique": 9,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'pop')",
"methods": 9,
"unique": 2,
"unique_ratio": 0.222,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('flask', 'json', 'if')",
"methods": 8,
"unique": 8,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'raises')",
"methods": 8,
"unique": 7,
"unique_ratio": 0.875,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'name_')",
"methods": 8,
"unique": 6,
"unique_ratio": 0.75,
"sore": "SKIP(large_alphabet)",
"sore_success": false
},
{
"context": "('tutorial', 'tests', 'login')",
"methods": 7,
"unique": 6,
"unique_ratio": 0.857,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('src', 'flask', 'super')",
"methods": 7,
"unique": 7,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'get')",
"methods": 7,
"unique": 6,
"unique_ratio": 0.857,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('src', 'flask', 'for')",
"methods": 6,
"unique": 6,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('src', 'flask', '_get_current_object')",
"methods": 6,
"unique": 6,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('flask', 'sansio', 'append')",
"methods": 6,
"unique": 2,
"unique_ratio": 0.333,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('flask', 'sansio', 'record_once')",
"methods": 6,
"unique": 1,
"unique_ratio": 0.167,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('tests', 'yield')",
"methods": 6,
"unique": 3,
"unique_ratio": 0.5,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('tests', 'if')",
"methods": 6,
"unique": 6,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'cho(')",
"methods": 6,
"unique": 2,
"unique_ratio": 0.333,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('tutorial', 'flaskr', 'if')",
"methods": 5,
"unique": 5,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tutorial', 'tests', 'return')",
"methods": 5,
"unique": 5,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('src', 'flask', 'raise')",
"methods": 5,
"unique": 3,
"unique_ratio": 0.6,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('flask', 'json', 'raise')",
"methods": 5,
"unique": 1,
"unique_ratio": 0.2,
"sore": "raise",
"sore_success": true
},
{
"context": "('tests', 'await')",
"methods": 5,
"unique": 1,
"unique_ratio": 0.2,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('tests', 'test_client')",
"methods": 5,
"unique": 5,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'mkdir')",
"methods": 5,
"unique": 4,
"unique_ratio": 0.8,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('tests', 'ef')",
"methods": 5,
"unique": 5,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'lask(')",
"methods": 5,
"unique": 4,
"unique_ratio": 0.8,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('tests', 'get_flashed_messages')",
"methods": 5,
"unique": 4,
"unique_ratio": 0.8,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('tutorial', 'tests', 'get')",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('src', 'flask', 'import')",
"methods": 4,
"unique": 3,
"unique_ratio": 0.75,
"sore": "SKIP(large_alphabet)",
"sore_success": false
},
{
"context": "('tests', 'write_text')",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'em(sys.')",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'setenv')",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'ain_m')",
"methods": 4,
"unique": 4,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tutorial', 'flaskr', 'get_db')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('src', 'flask', 'join')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('src', 'flask', 'def')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('src', 'flask', 'from')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('flask', 'sansio', 'raise')",
"methods": 3,
"unique": 2,
"unique_ratio": 0.667,
"sore": "raise.((NotImplementedError)+)?",
"sore_success": true
},
{
"context": "('flask', 'sansio', 'or')",
"methods": 3,
"unique": 1,
"unique_ratio": 0.333,
"sore": "or",
"sore_success": true
},
{
"context": "('flask', 'sansio', 'setdefault')",
"methods": 3,
"unique": 1,
"unique_ratio": 0.333,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('tests', 'app_context')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'is')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'setattr')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'rom')",
"methods": 3,
"unique": 2,
"unique_ratio": 0.667,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('tests', 'nvoke(')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'update')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'not')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 't')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tests', 'lask.u')",
"methods": 3,
"unique": 2,
"unique_ratio": 0.667,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('tests', 'ndpoin')",
"methods": 3,
"unique": 1,
"unique_ratio": 0.333,
"sore": "ndpoin",
"sore_success": true
}
],
"coverage": 2.2,
"elapsed_seconds": 0.08
}
Reference in a new issue View git blame Copy permalink