grammar-inference-engine/experiments/results/ragsak_summary.json
tobjend b516b2985d feat: implement Reduce algorithm (Algorithm 4, TODS 2010)
- bex/reduce.py: Faithful implementation of Reduce with support-weighted
  SOA edit distance, adjunction, iterative merging, and Minimize
- experiments/context_eval.py: Multi-codebase support (RAGSAK + Flask),
  Reduce experiments with thresholds 0.05-0.4
- tests/test_reduce.py: 24 tests covering all Reduce components
- Flask cloned to external_refs/flask for cross-validation

Results:
- RAGSAK: 12.0% coverage (First 3 symbols)
- Flask: 10.7% coverage (First 3 symbols)
- Reduce has minimal impact (1-2 merges per codebase at ε=0.3)
- Coverage ceiling appears to be ~10-12% for prefix-based grouping
2026-07-12 00:11:38 +02:00

754 lines
No EOL
18 KiB
JSON

[
{
"strategy": "Baseline: Package grouping",
"total_contexts": 137,
"meaningful_contexts": 116,
"total_methods": 1594,
"methods_in_good_groups": 9,
"sore_successes": 2,
"sore_failures": 10,
"skip_reasons": {
"too_large": 5,
"too_diverse": 89,
"large_alphabet": 10
},
"coverage": 0.6,
"elapsed_seconds": 0.07
},
{
"strategy": "Option A: File path k=1",
"total_contexts": 47,
"meaningful_contexts": 43,
"total_methods": 1594,
"methods_in_good_groups": 17,
"sore_successes": 2,
"sore_failures": 1,
"skip_reasons": {
"too_large": 9,
"too_diverse": 28,
"large_alphabet": 3
},
"coverage": 1.1,
"elapsed_seconds": 0.0
},
{
"strategy": "Option A: File path k=2",
"total_contexts": 59,
"meaningful_contexts": 54,
"total_methods": 1594,
"methods_in_good_groups": 17,
"sore_successes": 2,
"sore_failures": 1,
"skip_reasons": {
"too_large": 8,
"too_diverse": 39,
"large_alphabet": 4
},
"coverage": 1.1,
"elapsed_seconds": 0.01
},
{
"strategy": "Option A: File path k=3",
"total_contexts": 59,
"meaningful_contexts": 54,
"total_methods": 1594,
"methods_in_good_groups": 17,
"sore_successes": 2,
"sore_failures": 1,
"skip_reasons": {
"too_large": 8,
"too_diverse": 39,
"large_alphabet": 4
},
"coverage": 1.1,
"elapsed_seconds": 0.01
},
{
"strategy": "Option B: First 1 symbols",
"total_contexts": 550,
"meaningful_contexts": 134,
"total_methods": 1594,
"methods_in_good_groups": 74,
"sore_successes": 20,
"sore_failures": 19,
"skip_reasons": {
"too_large": 2,
"large_alphabet": 13,
"too_diverse": 80
},
"coverage": 4.6,
"elapsed_seconds": 0.08
},
{
"strategy": "Option B: First 2 symbols",
"total_contexts": 961,
"meaningful_contexts": 141,
"total_methods": 1594,
"methods_in_good_groups": 157,
"sore_successes": 39,
"sore_failures": 23,
"skip_reasons": {
"large_alphabet": 7,
"too_diverse": 72
},
"coverage": 9.8,
"elapsed_seconds": 0.13
},
{
"strategy": "Option B: First 3 symbols",
"total_contexts": 1112,
"meaningful_contexts": 117,
"total_methods": 1594,
"methods_in_good_groups": 191,
"sore_successes": 47,
"sore_failures": 14,
"skip_reasons": {
"large_alphabet": 6,
"too_diverse": 50
},
"coverage": 12.0,
"elapsed_seconds": 0.1
},
{
"strategy": "Option C: Path k=1 + Symbol k=1",
"total_contexts": 813,
"meaningful_contexts": 151,
"total_methods": 1594,
"methods_in_good_groups": 101,
"sore_successes": 24,
"sore_failures": 19,
"skip_reasons": {
"too_diverse": 99,
"large_alphabet": 9
},
"coverage": 6.3,
"elapsed_seconds": 0.1
},
{
"strategy": "Option C: Path k=1 + Symbol k=2",
"total_contexts": 1073,
"meaningful_contexts": 119,
"total_methods": 1594,
"methods_in_good_groups": 138,
"sore_successes": 33,
"sore_failures": 21,
"skip_reasons": {
"too_diverse": 61,
"large_alphabet": 4
},
"coverage": 8.7,
"elapsed_seconds": 0.07
},
{
"strategy": "Option C: Path k=2 + Symbol k=1",
"total_contexts": 832,
"meaningful_contexts": 146,
"total_methods": 1594,
"methods_in_good_groups": 101,
"sore_successes": 24,
"sore_failures": 19,
"skip_reasons": {
"too_diverse": 94,
"large_alphabet": 9
},
"coverage": 6.3,
"elapsed_seconds": 0.08
},
{
"strategy": "Option C: Path k=2 + Symbol k=2",
"total_contexts": 1080,
"meaningful_contexts": 117,
"total_methods": 1594,
"methods_in_good_groups": 138,
"sore_successes": 33,
"sore_failures": 21,
"skip_reasons": {
"too_diverse": 59,
"large_alphabet": 4
},
"coverage": 8.7,
"elapsed_seconds": 0.07
},
{
"strategy": "Option H: Return type heuristic",
"total_contexts": 60,
"meaningful_contexts": 5,
"total_methods": 1594,
"methods_in_good_groups": 0,
"sore_successes": 0,
"sore_failures": 0,
"skip_reasons": {
"too_large": 1,
"too_diverse": 4
},
"coverage": 0.0,
"elapsed_seconds": 0.0
},
{
"strategy": "Reduce k=1 \u03b5=0.05",
"total_contexts": 550,
"meaningful_contexts": 134,
"total_methods": 1594,
"methods_in_good_groups": 74,
"sore_successes": 20,
"sore_failures": 19,
"skip_reasons": {
"too_large": 2,
"large_alphabet": 13,
"too_diverse": 80
},
"coverage": 4.6,
"elapsed_seconds": 0.09,
"merge_info": {
"merges": {
"iterations": 1,
"merges": 0,
"comparisons": 23653,
"contexts_before": 550,
"contexts_after": 550,
"threshold": 0.05,
"merge_log": []
},
"threshold": 0.05,
"contexts_before": 550,
"contexts_after": 550
}
},
{
"strategy": "Reduce k=1 \u03b5=0.1",
"total_contexts": 550,
"meaningful_contexts": 134,
"total_methods": 1594,
"methods_in_good_groups": 74,
"sore_successes": 20,
"sore_failures": 19,
"skip_reasons": {
"too_large": 2,
"large_alphabet": 13,
"too_diverse": 80
},
"coverage": 4.6,
"elapsed_seconds": 0.08,
"merge_info": {
"merges": {
"iterations": 1,
"merges": 0,
"comparisons": 23653,
"contexts_before": 550,
"contexts_after": 550,
"threshold": 0.1,
"merge_log": []
},
"threshold": 0.1,
"contexts_before": 550,
"contexts_after": 550
}
},
{
"strategy": "Reduce k=1 \u03b5=0.15",
"total_contexts": 550,
"meaningful_contexts": 134,
"total_methods": 1594,
"methods_in_good_groups": 74,
"sore_successes": 20,
"sore_failures": 19,
"skip_reasons": {
"too_large": 2,
"large_alphabet": 13,
"too_diverse": 80
},
"coverage": 4.6,
"elapsed_seconds": 0.05,
"merge_info": {
"merges": {
"iterations": 1,
"merges": 0,
"comparisons": 23653,
"contexts_before": 550,
"contexts_after": 550,
"threshold": 0.15,
"merge_log": []
},
"threshold": 0.15,
"contexts_before": 550,
"contexts_after": 550
}
},
{
"strategy": "Reduce k=1 \u03b5=0.2",
"total_contexts": 550,
"meaningful_contexts": 134,
"total_methods": 1594,
"methods_in_good_groups": 74,
"sore_successes": 20,
"sore_failures": 19,
"skip_reasons": {
"too_large": 2,
"large_alphabet": 13,
"too_diverse": 80
},
"coverage": 4.6,
"elapsed_seconds": 0.13,
"merge_info": {
"merges": {
"iterations": 1,
"merges": 0,
"comparisons": 23653,
"contexts_before": 550,
"contexts_after": 550,
"threshold": 0.2,
"merge_log": []
},
"threshold": 0.2,
"contexts_before": 550,
"contexts_after": 550
}
},
{
"strategy": "Reduce k=1 \u03b5=0.3",
"total_contexts": 550,
"meaningful_contexts": 134,
"total_methods": 1594,
"methods_in_good_groups": 74,
"sore_successes": 20,
"sore_failures": 19,
"skip_reasons": {
"too_large": 2,
"large_alphabet": 13,
"too_diverse": 80
},
"coverage": 4.6,
"elapsed_seconds": 0.05,
"merge_info": {
"merges": {
"iterations": 1,
"merges": 0,
"comparisons": 23653,
"contexts_before": 550,
"contexts_after": 550,
"threshold": 0.3,
"merge_log": []
},
"threshold": 0.3,
"contexts_before": 550,
"contexts_after": 550
}
},
{
"strategy": "Reduce k=1 \u03b5=0.4",
"total_contexts": 550,
"meaningful_contexts": 134,
"total_methods": 1594,
"methods_in_good_groups": 74,
"sore_successes": 20,
"sore_failures": 19,
"skip_reasons": {
"too_large": 2,
"large_alphabet": 13,
"too_diverse": 80
},
"coverage": 4.6,
"elapsed_seconds": 0.05,
"merge_info": {
"merges": {
"iterations": 1,
"merges": 0,
"comparisons": 23653,
"contexts_before": 550,
"contexts_after": 550,
"threshold": 0.4,
"merge_log": []
},
"threshold": 0.4,
"contexts_before": 550,
"contexts_after": 550
}
},
{
"strategy": "Reduce k=2 \u03b5=0.05",
"total_contexts": 961,
"meaningful_contexts": 141,
"total_methods": 1594,
"methods_in_good_groups": 157,
"sore_successes": 39,
"sore_failures": 23,
"skip_reasons": {
"large_alphabet": 7,
"too_diverse": 72
},
"coverage": 9.8,
"elapsed_seconds": 0.06,
"merge_info": {
"merges": {
"iterations": 1,
"merges": 0,
"comparisons": 32896,
"contexts_before": 961,
"contexts_after": 961,
"threshold": 0.05,
"merge_log": []
},
"threshold": 0.05,
"contexts_before": 961,
"contexts_after": 961
}
},
{
"strategy": "Reduce k=2 \u03b5=0.1",
"total_contexts": 961,
"meaningful_contexts": 141,
"total_methods": 1594,
"methods_in_good_groups": 157,
"sore_successes": 39,
"sore_failures": 23,
"skip_reasons": {
"large_alphabet": 7,
"too_diverse": 72
},
"coverage": 9.8,
"elapsed_seconds": 0.12,
"merge_info": {
"merges": {
"iterations": 1,
"merges": 0,
"comparisons": 32896,
"contexts_before": 961,
"contexts_after": 961,
"threshold": 0.1,
"merge_log": []
},
"threshold": 0.1,
"contexts_before": 961,
"contexts_after": 961
}
},
{
"strategy": "Reduce k=2 \u03b5=0.15",
"total_contexts": 961,
"meaningful_contexts": 141,
"total_methods": 1594,
"methods_in_good_groups": 157,
"sore_successes": 39,
"sore_failures": 23,
"skip_reasons": {
"large_alphabet": 7,
"too_diverse": 72
},
"coverage": 9.8,
"elapsed_seconds": 0.15,
"merge_info": {
"merges": {
"iterations": 1,
"merges": 0,
"comparisons": 32896,
"contexts_before": 961,
"contexts_after": 961,
"threshold": 0.15,
"merge_log": []
},
"threshold": 0.15,
"contexts_before": 961,
"contexts_after": 961
}
},
{
"strategy": "Reduce k=2 \u03b5=0.2",
"total_contexts": 961,
"meaningful_contexts": 141,
"total_methods": 1594,
"methods_in_good_groups": 157,
"sore_successes": 39,
"sore_failures": 23,
"skip_reasons": {
"large_alphabet": 7,
"too_diverse": 72
},
"coverage": 9.8,
"elapsed_seconds": 0.13,
"merge_info": {
"merges": {
"iterations": 1,
"merges": 0,
"comparisons": 32896,
"contexts_before": 961,
"contexts_after": 961,
"threshold": 0.2,
"merge_log": []
},
"threshold": 0.2,
"contexts_before": 961,
"contexts_after": 961
}
},
{
"strategy": "Reduce k=2 \u03b5=0.3",
"total_contexts": 961,
"meaningful_contexts": 141,
"total_methods": 1597,
"methods_in_good_groups": 153,
"sore_successes": 38,
"sore_failures": 24,
"skip_reasons": {
"large_alphabet": 7,
"too_diverse": 72
},
"coverage": 9.6,
"elapsed_seconds": 0.1,
"merge_info": {
"merges": {
"iterations": 2,
"merges": 1,
"comparisons": 65536,
"contexts_before": 961,
"contexts_after": 961,
"threshold": 0.3,
"merge_log": [
{
"iteration": 1,
"merged_into": "('JobStatus', 'every')",
"removed": "('JobStatus', 'now')",
"distance": 0.2714,
"new_size": 7
}
]
},
"threshold": 0.3,
"contexts_before": 961,
"contexts_after": 961
}
},
{
"strategy": "Reduce k=2 \u03b5=0.4",
"total_contexts": 961,
"meaningful_contexts": 141,
"total_methods": 1604,
"methods_in_good_groups": 153,
"sore_successes": 38,
"sore_failures": 24,
"skip_reasons": {
"large_alphabet": 7,
"too_diverse": 72
},
"coverage": 9.5,
"elapsed_seconds": 0.1,
"merge_info": {
"merges": {
"iterations": 2,
"merges": 2,
"comparisons": 65281,
"contexts_before": 961,
"contexts_after": 961,
"threshold": 0.4,
"merge_log": [
{
"iteration": 1,
"merged_into": "('JobStatus', 'every')",
"removed": "('every', 'getJobStatus')",
"distance": 0.3853,
"new_size": 11
},
{
"iteration": 1,
"merged_into": "('JobStatus', 'every')",
"removed": "('JobStatus', 'now')",
"distance": 0.2714,
"new_size": 14
}
]
},
"threshold": 0.4,
"contexts_before": 961,
"contexts_after": 961
}
},
{
"strategy": "Reduce k=3 \u03b5=0.05",
"total_contexts": 1112,
"meaningful_contexts": 117,
"total_methods": 1594,
"methods_in_good_groups": 191,
"sore_successes": 47,
"sore_failures": 14,
"skip_reasons": {
"large_alphabet": 6,
"too_diverse": 50
},
"coverage": 12.0,
"elapsed_seconds": 0.09,
"merge_info": {
"merges": {
"iterations": 1,
"merges": 0,
"comparisons": 27495,
"contexts_before": 1112,
"contexts_after": 1112,
"threshold": 0.05,
"merge_log": []
},
"threshold": 0.05,
"contexts_before": 1112,
"contexts_after": 1112
}
},
{
"strategy": "Reduce k=3 \u03b5=0.1",
"total_contexts": 1112,
"meaningful_contexts": 117,
"total_methods": 1594,
"methods_in_good_groups": 191,
"sore_successes": 47,
"sore_failures": 14,
"skip_reasons": {
"large_alphabet": 6,
"too_diverse": 50
},
"coverage": 12.0,
"elapsed_seconds": 0.12,
"merge_info": {
"merges": {
"iterations": 1,
"merges": 0,
"comparisons": 27495,
"contexts_before": 1112,
"contexts_after": 1112,
"threshold": 0.1,
"merge_log": []
},
"threshold": 0.1,
"contexts_before": 1112,
"contexts_after": 1112
}
},
{
"strategy": "Reduce k=3 \u03b5=0.15",
"total_contexts": 1112,
"meaningful_contexts": 117,
"total_methods": 1594,
"methods_in_good_groups": 191,
"sore_successes": 47,
"sore_failures": 14,
"skip_reasons": {
"large_alphabet": 6,
"too_diverse": 50
},
"coverage": 12.0,
"elapsed_seconds": 0.04,
"merge_info": {
"merges": {
"iterations": 1,
"merges": 0,
"comparisons": 27495,
"contexts_before": 1112,
"contexts_after": 1112,
"threshold": 0.15,
"merge_log": []
},
"threshold": 0.15,
"contexts_before": 1112,
"contexts_after": 1112
}
},
{
"strategy": "Reduce k=3 \u03b5=0.2",
"total_contexts": 1112,
"meaningful_contexts": 117,
"total_methods": 1594,
"methods_in_good_groups": 191,
"sore_successes": 47,
"sore_failures": 14,
"skip_reasons": {
"large_alphabet": 6,
"too_diverse": 50
},
"coverage": 12.0,
"elapsed_seconds": 0.09,
"merge_info": {
"merges": {
"iterations": 1,
"merges": 0,
"comparisons": 27495,
"contexts_before": 1112,
"contexts_after": 1112,
"threshold": 0.2,
"merge_log": []
},
"threshold": 0.2,
"contexts_before": 1112,
"contexts_after": 1112
}
},
{
"strategy": "Reduce k=3 \u03b5=0.3",
"total_contexts": 1112,
"meaningful_contexts": 117,
"total_methods": 1597,
"methods_in_good_groups": 187,
"sore_successes": 46,
"sore_failures": 15,
"skip_reasons": {
"large_alphabet": 6,
"too_diverse": 50
},
"coverage": 11.7,
"elapsed_seconds": 0.07,
"merge_info": {
"merges": {
"iterations": 2,
"merges": 1,
"comparisons": 54756,
"contexts_before": 1112,
"contexts_after": 1112,
"threshold": 0.3,
"merge_log": [
{
"iteration": 1,
"merged_into": "('JobStatus', 'every', 'getJobStatus')",
"removed": "('JobStatus', 'now', 'minusMinutes')",
"distance": 0.2714,
"new_size": 7
}
]
},
"threshold": 0.3,
"contexts_before": 1112,
"contexts_after": 1112
}
},
{
"strategy": "Reduce k=3 \u03b5=0.4",
"total_contexts": 1112,
"meaningful_contexts": 117,
"total_methods": 1597,
"methods_in_good_groups": 187,
"sore_successes": 46,
"sore_failures": 15,
"skip_reasons": {
"large_alphabet": 6,
"too_diverse": 50
},
"coverage": 11.7,
"elapsed_seconds": 0.04,
"merge_info": {
"merges": {
"iterations": 2,
"merges": 1,
"comparisons": 54756,
"contexts_before": 1112,
"contexts_after": 1112,
"threshold": 0.4,
"merge_log": [
{
"iteration": 1,
"merged_into": "('JobStatus', 'every', 'getJobStatus')",
"removed": "('JobStatus', 'now', 'minusMinutes')",
"distance": 0.2714,
"new_size": 7
}
]
},
"threshold": 0.4,
"contexts_before": 1112,
"contexts_after": 1112
}
}
]