grammar-inference-engine/experiments/results/flask_summary.json
tobjend b516b2985d feat: implement Reduce algorithm (Algorithm 4, TODS 2010)
- bex/reduce.py: Faithful implementation of Reduce with support-weighted
  SOA edit distance, adjunction, iterative merging, and Minimize
- experiments/context_eval.py: Multi-codebase support (RAGSAK + Flask),
  Reduce experiments with thresholds 0.05-0.4
- tests/test_reduce.py: 24 tests covering all Reduce components
- Flask cloned to external_refs/flask for cross-validation

Results:
- RAGSAK: 12.0% coverage (First 3 symbols)
- Flask: 10.7% coverage (First 3 symbols)
- Reduce has minimal impact (1-2 merges per codebase at ε=0.3)
- Coverage ceiling appears to be ~10-12% for prefix-based grouping
2026-07-12 00:11:38 +02:00

764 lines
No EOL
18 KiB
JSON

[
{
"strategy": "Baseline: Package grouping",
"total_contexts": 15,
"meaningful_contexts": 9,
"total_methods": 1391,
"methods_in_good_groups": 0,
"sore_successes": 0,
"sore_failures": 1,
"skip_reasons": {
"too_large": 3,
"large_alphabet": 1,
"too_diverse": 4
},
"coverage": 0.0,
"elapsed_seconds": 0.02
},
{
"strategy": "Option A: File path k=1",
"total_contexts": 13,
"meaningful_contexts": 7,
"total_methods": 1391,
"methods_in_good_groups": 0,
"sore_successes": 0,
"sore_failures": 1,
"skip_reasons": {
"too_large": 3,
"large_alphabet": 1,
"too_diverse": 2
},
"coverage": 0.0,
"elapsed_seconds": 0.01
},
{
"strategy": "Option A: File path k=2",
"total_contexts": 15,
"meaningful_contexts": 9,
"total_methods": 1391,
"methods_in_good_groups": 0,
"sore_successes": 0,
"sore_failures": 1,
"skip_reasons": {
"too_large": 3,
"large_alphabet": 1,
"too_diverse": 4
},
"coverage": 0.0,
"elapsed_seconds": 0.02
},
{
"strategy": "Option A: File path k=3",
"total_contexts": 15,
"meaningful_contexts": 9,
"total_methods": 1391,
"methods_in_good_groups": 0,
"sore_successes": 0,
"sore_failures": 1,
"skip_reasons": {
"too_large": 3,
"large_alphabet": 1,
"too_diverse": 4
},
"coverage": 0.0,
"elapsed_seconds": 0.01
},
{
"strategy": "Option B: First 1 symbols",
"total_contexts": 228,
"meaningful_contexts": 53,
"total_methods": 1391,
"methods_in_good_groups": 19,
"sore_successes": 2,
"sore_failures": 14,
"skip_reasons": {
"too_large": 5,
"too_diverse": 29,
"large_alphabet": 3
},
"coverage": 1.4,
"elapsed_seconds": 0.08
},
{
"strategy": "Option B: First 2 symbols",
"total_contexts": 430,
"meaningful_contexts": 96,
"total_methods": 1391,
"methods_in_good_groups": 102,
"sore_successes": 18,
"sore_failures": 23,
"skip_reasons": {
"too_large": 3,
"too_diverse": 50,
"large_alphabet": 2
},
"coverage": 7.3,
"elapsed_seconds": 0.08
},
{
"strategy": "Option B: First 3 symbols",
"total_contexts": 598,
"meaningful_contexts": 85,
"total_methods": 1391,
"methods_in_good_groups": 149,
"sore_successes": 21,
"sore_failures": 23,
"skip_reasons": {
"too_large": 2,
"too_diverse": 38,
"large_alphabet": 1
},
"coverage": 10.7,
"elapsed_seconds": 0.11
},
{
"strategy": "Option C: Path k=1 + Symbol k=1",
"total_contexts": 288,
"meaningful_contexts": 68,
"total_methods": 1391,
"methods_in_good_groups": 30,
"sore_successes": 5,
"sore_failures": 19,
"skip_reasons": {
"too_large": 3,
"large_alphabet": 5,
"too_diverse": 36
},
"coverage": 2.2,
"elapsed_seconds": 0.07
},
{
"strategy": "Option C: Path k=1 + Symbol k=2",
"total_contexts": 512,
"meaningful_contexts": 96,
"total_methods": 1391,
"methods_in_good_groups": 141,
"sore_successes": 22,
"sore_failures": 22,
"skip_reasons": {
"too_large": 2,
"too_diverse": 48,
"large_alphabet": 2
},
"coverage": 10.1,
"elapsed_seconds": 0.07
},
{
"strategy": "Option C: Path k=2 + Symbol k=1",
"total_contexts": 297,
"meaningful_contexts": 69,
"total_methods": 1391,
"methods_in_good_groups": 30,
"sore_successes": 5,
"sore_failures": 19,
"skip_reasons": {
"too_large": 3,
"large_alphabet": 5,
"too_diverse": 37
},
"coverage": 2.2,
"elapsed_seconds": 0.08
},
{
"strategy": "Option C: Path k=2 + Symbol k=2",
"total_contexts": 520,
"meaningful_contexts": 93,
"total_methods": 1391,
"methods_in_good_groups": 132,
"sore_successes": 19,
"sore_failures": 22,
"skip_reasons": {
"too_large": 2,
"too_diverse": 48,
"large_alphabet": 2
},
"coverage": 9.5,
"elapsed_seconds": 0.04
},
{
"strategy": "Option H: Return type heuristic",
"total_contexts": 3,
"meaningful_contexts": 3,
"total_methods": 1391,
"methods_in_good_groups": 0,
"sore_successes": 0,
"sore_failures": 1,
"skip_reasons": {
"too_large": 2
},
"coverage": 0.0,
"elapsed_seconds": 0.0
},
{
"strategy": "Reduce k=1 \u03b5=0.05",
"total_contexts": 228,
"meaningful_contexts": 53,
"total_methods": 1391,
"methods_in_good_groups": 19,
"sore_successes": 2,
"sore_failures": 14,
"skip_reasons": {
"too_large": 5,
"too_diverse": 29,
"large_alphabet": 3
},
"coverage": 1.4,
"elapsed_seconds": 0.05,
"merge_info": {
"merges": {
"iterations": 1,
"merges": 0,
"comparisons": 4095,
"contexts_before": 228,
"contexts_after": 228,
"threshold": 0.05,
"merge_log": []
},
"threshold": 0.05,
"contexts_before": 228,
"contexts_after": 228
}
},
{
"strategy": "Reduce k=1 \u03b5=0.1",
"total_contexts": 228,
"meaningful_contexts": 53,
"total_methods": 1391,
"methods_in_good_groups": 19,
"sore_successes": 2,
"sore_failures": 14,
"skip_reasons": {
"too_large": 5,
"too_diverse": 29,
"large_alphabet": 3
},
"coverage": 1.4,
"elapsed_seconds": 0.03,
"merge_info": {
"merges": {
"iterations": 1,
"merges": 0,
"comparisons": 4095,
"contexts_before": 228,
"contexts_after": 228,
"threshold": 0.1,
"merge_log": []
},
"threshold": 0.1,
"contexts_before": 228,
"contexts_after": 228
}
},
{
"strategy": "Reduce k=1 \u03b5=0.15",
"total_contexts": 228,
"meaningful_contexts": 53,
"total_methods": 1391,
"methods_in_good_groups": 19,
"sore_successes": 2,
"sore_failures": 14,
"skip_reasons": {
"too_large": 5,
"too_diverse": 29,
"large_alphabet": 3
},
"coverage": 1.4,
"elapsed_seconds": 0.06,
"merge_info": {
"merges": {
"iterations": 1,
"merges": 0,
"comparisons": 4095,
"contexts_before": 228,
"contexts_after": 228,
"threshold": 0.15,
"merge_log": []
},
"threshold": 0.15,
"contexts_before": 228,
"contexts_after": 228
}
},
{
"strategy": "Reduce k=1 \u03b5=0.2",
"total_contexts": 228,
"meaningful_contexts": 53,
"total_methods": 1391,
"methods_in_good_groups": 19,
"sore_successes": 2,
"sore_failures": 14,
"skip_reasons": {
"too_large": 5,
"too_diverse": 29,
"large_alphabet": 3
},
"coverage": 1.4,
"elapsed_seconds": 0.07,
"merge_info": {
"merges": {
"iterations": 1,
"merges": 0,
"comparisons": 4095,
"contexts_before": 228,
"contexts_after": 228,
"threshold": 0.2,
"merge_log": []
},
"threshold": 0.2,
"contexts_before": 228,
"contexts_after": 228
}
},
{
"strategy": "Reduce k=1 \u03b5=0.3",
"total_contexts": 228,
"meaningful_contexts": 53,
"total_methods": 1391,
"methods_in_good_groups": 19,
"sore_successes": 2,
"sore_failures": 14,
"skip_reasons": {
"too_large": 5,
"too_diverse": 29,
"large_alphabet": 3
},
"coverage": 1.4,
"elapsed_seconds": 0.06,
"merge_info": {
"merges": {
"iterations": 1,
"merges": 0,
"comparisons": 4095,
"contexts_before": 228,
"contexts_after": 228,
"threshold": 0.3,
"merge_log": []
},
"threshold": 0.3,
"contexts_before": 228,
"contexts_after": 228
}
},
{
"strategy": "Reduce k=1 \u03b5=0.4",
"total_contexts": 228,
"meaningful_contexts": 53,
"total_methods": 1391,
"methods_in_good_groups": 19,
"sore_successes": 2,
"sore_failures": 14,
"skip_reasons": {
"too_large": 5,
"too_diverse": 29,
"large_alphabet": 3
},
"coverage": 1.4,
"elapsed_seconds": 0.04,
"merge_info": {
"merges": {
"iterations": 1,
"merges": 0,
"comparisons": 4095,
"contexts_before": 228,
"contexts_after": 228,
"threshold": 0.4,
"merge_log": []
},
"threshold": 0.4,
"contexts_before": 228,
"contexts_after": 228
}
},
{
"strategy": "Reduce k=2 \u03b5=0.05",
"total_contexts": 430,
"meaningful_contexts": 96,
"total_methods": 1391,
"methods_in_good_groups": 102,
"sore_successes": 18,
"sore_failures": 23,
"skip_reasons": {
"too_large": 3,
"too_diverse": 50,
"large_alphabet": 2
},
"coverage": 7.3,
"elapsed_seconds": 0.07,
"merge_info": {
"merges": {
"iterations": 1,
"merges": 0,
"comparisons": 11026,
"contexts_before": 430,
"contexts_after": 430,
"threshold": 0.05,
"merge_log": []
},
"threshold": 0.05,
"contexts_before": 430,
"contexts_after": 430
}
},
{
"strategy": "Reduce k=2 \u03b5=0.1",
"total_contexts": 430,
"meaningful_contexts": 96,
"total_methods": 1391,
"methods_in_good_groups": 102,
"sore_successes": 18,
"sore_failures": 23,
"skip_reasons": {
"too_large": 3,
"too_diverse": 50,
"large_alphabet": 2
},
"coverage": 7.3,
"elapsed_seconds": 0.05,
"merge_info": {
"merges": {
"iterations": 1,
"merges": 0,
"comparisons": 11026,
"contexts_before": 430,
"contexts_after": 430,
"threshold": 0.1,
"merge_log": []
},
"threshold": 0.1,
"contexts_before": 430,
"contexts_after": 430
}
},
{
"strategy": "Reduce k=2 \u03b5=0.15",
"total_contexts": 430,
"meaningful_contexts": 96,
"total_methods": 1391,
"methods_in_good_groups": 102,
"sore_successes": 18,
"sore_failures": 23,
"skip_reasons": {
"too_large": 3,
"too_diverse": 50,
"large_alphabet": 2
},
"coverage": 7.3,
"elapsed_seconds": 0.05,
"merge_info": {
"merges": {
"iterations": 1,
"merges": 0,
"comparisons": 11026,
"contexts_before": 430,
"contexts_after": 430,
"threshold": 0.15,
"merge_log": []
},
"threshold": 0.15,
"contexts_before": 430,
"contexts_after": 430
}
},
{
"strategy": "Reduce k=2 \u03b5=0.2",
"total_contexts": 430,
"meaningful_contexts": 96,
"total_methods": 1391,
"methods_in_good_groups": 102,
"sore_successes": 18,
"sore_failures": 23,
"skip_reasons": {
"too_large": 3,
"too_diverse": 50,
"large_alphabet": 2
},
"coverage": 7.3,
"elapsed_seconds": 0.12,
"merge_info": {
"merges": {
"iterations": 1,
"merges": 0,
"comparisons": 11026,
"contexts_before": 430,
"contexts_after": 430,
"threshold": 0.2,
"merge_log": []
},
"threshold": 0.2,
"contexts_before": 430,
"contexts_after": 430
}
},
{
"strategy": "Reduce k=2 \u03b5=0.3",
"total_contexts": 430,
"meaningful_contexts": 97,
"total_methods": 1393,
"methods_in_good_groups": 102,
"sore_successes": 18,
"sore_failures": 23,
"skip_reasons": {
"too_large": 3,
"too_diverse": 51,
"large_alphabet": 2
},
"coverage": 7.3,
"elapsed_seconds": 0.05,
"merge_info": {
"merges": {
"iterations": 2,
"merges": 1,
"comparisons": 21904,
"contexts_before": 430,
"contexts_after": 430,
"threshold": 0.3,
"merge_log": [
{
"iteration": 1,
"merged_into": "('def', 'boolean')",
"removed": "('def', 'is_boolean')",
"distance": 0.2424,
"new_size": 4
}
]
},
"threshold": 0.3,
"contexts_before": 430,
"contexts_after": 430
}
},
{
"strategy": "Reduce k=2 \u03b5=0.4",
"total_contexts": 430,
"meaningful_contexts": 97,
"total_methods": 1393,
"methods_in_good_groups": 102,
"sore_successes": 18,
"sore_failures": 23,
"skip_reasons": {
"too_large": 3,
"too_diverse": 51,
"large_alphabet": 2
},
"coverage": 7.3,
"elapsed_seconds": 0.05,
"merge_info": {
"merges": {
"iterations": 2,
"merges": 1,
"comparisons": 21904,
"contexts_before": 430,
"contexts_after": 430,
"threshold": 0.4,
"merge_log": [
{
"iteration": 1,
"merged_into": "('def', 'boolean')",
"removed": "('def', 'is_boolean')",
"distance": 0.2424,
"new_size": 4
}
]
},
"threshold": 0.4,
"contexts_before": 430,
"contexts_after": 430
}
},
{
"strategy": "Reduce k=3 \u03b5=0.05",
"total_contexts": 598,
"meaningful_contexts": 85,
"total_methods": 1391,
"methods_in_good_groups": 149,
"sore_successes": 21,
"sore_failures": 23,
"skip_reasons": {
"too_large": 2,
"too_diverse": 38,
"large_alphabet": 1
},
"coverage": 10.7,
"elapsed_seconds": 0.05,
"merge_info": {
"merges": {
"iterations": 1,
"merges": 0,
"comparisons": 11325,
"contexts_before": 598,
"contexts_after": 598,
"threshold": 0.05,
"merge_log": []
},
"threshold": 0.05,
"contexts_before": 598,
"contexts_after": 598
}
},
{
"strategy": "Reduce k=3 \u03b5=0.1",
"total_contexts": 598,
"meaningful_contexts": 85,
"total_methods": 1391,
"methods_in_good_groups": 149,
"sore_successes": 21,
"sore_failures": 23,
"skip_reasons": {
"too_large": 2,
"too_diverse": 38,
"large_alphabet": 1
},
"coverage": 10.7,
"elapsed_seconds": 0.05,
"merge_info": {
"merges": {
"iterations": 1,
"merges": 0,
"comparisons": 11325,
"contexts_before": 598,
"contexts_after": 598,
"threshold": 0.1,
"merge_log": []
},
"threshold": 0.1,
"contexts_before": 598,
"contexts_after": 598
}
},
{
"strategy": "Reduce k=3 \u03b5=0.15",
"total_contexts": 598,
"meaningful_contexts": 85,
"total_methods": 1391,
"methods_in_good_groups": 149,
"sore_successes": 21,
"sore_failures": 23,
"skip_reasons": {
"too_large": 2,
"too_diverse": 38,
"large_alphabet": 1
},
"coverage": 10.7,
"elapsed_seconds": 0.09,
"merge_info": {
"merges": {
"iterations": 1,
"merges": 0,
"comparisons": 11325,
"contexts_before": 598,
"contexts_after": 598,
"threshold": 0.15,
"merge_log": []
},
"threshold": 0.15,
"contexts_before": 598,
"contexts_after": 598
}
},
{
"strategy": "Reduce k=3 \u03b5=0.2",
"total_contexts": 598,
"meaningful_contexts": 85,
"total_methods": 1391,
"methods_in_good_groups": 149,
"sore_successes": 21,
"sore_failures": 23,
"skip_reasons": {
"too_large": 2,
"too_diverse": 38,
"large_alphabet": 1
},
"coverage": 10.7,
"elapsed_seconds": 0.11,
"merge_info": {
"merges": {
"iterations": 1,
"merges": 0,
"comparisons": 11325,
"contexts_before": 598,
"contexts_after": 598,
"threshold": 0.2,
"merge_log": []
},
"threshold": 0.2,
"contexts_before": 598,
"contexts_after": 598
}
},
{
"strategy": "Reduce k=3 \u03b5=0.3",
"total_contexts": 598,
"meaningful_contexts": 86,
"total_methods": 1393,
"methods_in_good_groups": 149,
"sore_successes": 21,
"sore_failures": 23,
"skip_reasons": {
"too_large": 2,
"too_diverse": 39,
"large_alphabet": 1
},
"coverage": 10.7,
"elapsed_seconds": 0.11,
"merge_info": {
"merges": {
"iterations": 2,
"merges": 1,
"comparisons": 22500,
"contexts_before": 598,
"contexts_after": 598,
"threshold": 0.3,
"merge_log": [
{
"iteration": 1,
"merged_into": "('def', 'boolean', 'return')",
"removed": "('def', 'is_boolean', 'return')",
"distance": 0.2424,
"new_size": 4
}
]
},
"threshold": 0.3,
"contexts_before": 598,
"contexts_after": 598
}
},
{
"strategy": "Reduce k=3 \u03b5=0.4",
"total_contexts": 598,
"meaningful_contexts": 86,
"total_methods": 1393,
"methods_in_good_groups": 149,
"sore_successes": 21,
"sore_failures": 23,
"skip_reasons": {
"too_large": 2,
"too_diverse": 39,
"large_alphabet": 1
},
"coverage": 10.7,
"elapsed_seconds": 0.09,
"merge_info": {
"merges": {
"iterations": 2,
"merges": 1,
"comparisons": 22500,
"contexts_before": 598,
"contexts_after": 598,
"threshold": 0.4,
"merge_log": [
{
"iteration": 1,
"merged_into": "('def', 'boolean', 'return')",
"removed": "('def', 'is_boolean', 'return')",
"distance": 0.2424,
"new_size": 4
}
]
},
"threshold": 0.4,
"contexts_before": 598,
"contexts_after": 598
}
}
]