tobi/grammar-inference-engine
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions
grammar-inference-engine/experiments/results/ragsak_file_path_k2.json
tobjend b516b2985d feat: implement Reduce algorithm (Algorithm 4, TODS 2010) 
- bex/reduce.py: Faithful implementation of Reduce with support-weighted
  SOA edit distance, adjunction, iterative merging, and Minimize
- experiments/context_eval.py: Multi-codebase support (RAGSAK + Flask),
  Reduce experiments with thresholds 0.05-0.4
- tests/test_reduce.py: 24 tests covering all Reduce components
- Flask cloned to external_refs/flask for cross-validation

Results:
- RAGSAK: 12.0% coverage (First 3 symbols)
- Flask: 10.7% coverage (First 3 symbols)
- Reduce has minimal impact (1-2 merges per codebase at ε=0.3)
- Coverage ceiling appears to be ~10-12% for prefix-based grouping
2026-07-12 00:11:38 +02:00

450 lines
No EOL
10 KiB
JSON
Raw Blame History

{
"strategy": "Option A: File path k=2",
"total_contexts": 59,
"meaningful_contexts": 54,
"total_methods": 1594,
"methods_in_good_groups": 17,
"sore_successes": 2,
"sore_failures": 1,
"skip_reasons": {
"too_large": 8,
"too_diverse": 39,
"large_alphabet": 4
},
"groups": [
{
"context": "('springrag', 'controller')",
"methods": 144,
"unique": 132,
"unique_ratio": 0.917,
"sore": "SKIP(too_large)",
"sore_success": false
},
{
"context": "('service', 'job')",
"methods": 143,
"unique": 133,
"unique_ratio": 0.93,
"sore": "SKIP(too_large)",
"sore_success": false
},
{
"context": "('springrag', 'config')",
"methods": 128,
"unique": 85,
"unique_ratio": 0.664,
"sore": "SKIP(too_large)",
"sore_success": false
},
{
"context": "('rag', 'embabel')",
"methods": 87,
"unique": 77,
"unique_ratio": 0.885,
"sore": "SKIP(too_large)",
"sore_success": false
},
{
"context": "('springrag', 'architecture')",
"methods": 87,
"unique": 70,
"unique_ratio": 0.805,
"sore": "SKIP(too_large)",
"sore_success": false
},
{
"context": "('port', 'adapter')",
"methods": 76,
"unique": 74,
"unique_ratio": 0.974,
"sore": "SKIP(too_large)",
"sore_success": false
},
{
"context": "('controller', 'librechat')",
"methods": 75,
"unique": 69,
"unique_ratio": 0.92,
"sore": "SKIP(too_large)",
"sore_success": false
},
{
"context": "('springrag', 'system')",
"methods": 58,
"unique": 50,
"unique_ratio": 0.862,
"sore": "SKIP(too_large)",
"sore_success": false
},
{
"context": "('batch', 'listener')",
"methods": 50,
"unique": 44,
"unique_ratio": 0.88,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('springrag', 'repository')",
"methods": 43,
"unique": 39,
"unique_ratio": 0.907,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('service', 'storage')",
"methods": 43,
"unique": 29,
"unique_ratio": 0.674,
"sore": "SKIP(large_alphabet)",
"sore_success": false
},
{
"context": "('service', 'graph')",
"methods": 42,
"unique": 37,
"unique_ratio": 0.881,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('springrag', 'batch')",
"methods": 32,
"unique": 32,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('service', 'docling')",
"methods": 31,
"unique": 28,
"unique_ratio": 0.903,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('service', 'knowledgebase')",
"methods": 27,
"unique": 27,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('summarizer', 'embabel')",
"methods": 26,
"unique": 26,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('common', 'ids')",
"methods": 26,
"unique": 16,
"unique_ratio": 0.615,
"sore": "SKIP(large_alphabet)",
"sore_success": false
},
{
"context": "('service', 'chunk')",
"methods": 25,
"unique": 25,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('security', 'service')",
"methods": 25,
"unique": 22,
"unique_ratio": 0.88,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('springrag', 'health')",
"methods": 24,
"unique": 22,
"unique_ratio": 0.917,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('rag', 'simple')",
"methods": 23,
"unique": 23,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('rag', 'support')",
"methods": 23,
"unique": 22,
"unique_ratio": 0.957,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('repository', 'graph')",
"methods": 23,
"unique": 22,
"unique_ratio": 0.957,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('agent', 'wikipedia')",
"methods": 18,
"unique": 17,
"unique_ratio": 0.944,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tooling', 'embabel')",
"methods": 18,
"unique": 18,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('service', 'chat')",
"methods": 18,
"unique": 17,
"unique_ratio": 0.944,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('springrag', 'testcontainers')",
"methods": 18,
"unique": 18,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('cucumber', 'steps')",
"methods": 17,
"unique": 13,
"unique_ratio": 0.765,
"sore": "SKIP(large_alphabet)",
"sore_success": false
},
{
"context": "('batch', 'writer')",
"methods": 17,
"unique": 17,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('controller', 'web')",
"methods": 16,
"unique": 16,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('model', 'graph')",
"methods": 16,
"unique": 15,
"unique_ratio": 0.938,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('batch', 'reader')",
"methods": 16,
"unique": 15,
"unique_ratio": 0.938,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('embabel', 'librechat')",
"methods": 14,
"unique": 14,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('springrag', 'testsupport')",
"methods": 12,
"unique": 3,
"unique_ratio": 0.25,
"sore": "(get|(Builder|Any))",
"sore_success": true
},
{
"context": "('batch', 'model')",
"methods": 12,
"unique": 10,
"unique_ratio": 0.833,
"sore": "SKIP(large_alphabet)",
"sore_success": false
},
{
"context": "('capability', 'support')",
"methods": 11,
"unique": 10,
"unique_ratio": 0.909,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('springrag', 'model')",
"methods": 11,
"unique": 11,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('embabel', 'recording')",
"methods": 11,
"unique": 11,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('batch', 'partition')",
"methods": 11,
"unique": 11,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('security', 'config')",
"methods": 11,
"unique": 11,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('main', 'kotlin')",
"methods": 8,
"unique": 8,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('service', 'cleanup')",
"methods": 8,
"unique": 6,
"unique_ratio": 0.75,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('agent', 'rag')",
"methods": 7,
"unique": 7,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('controller', 'auth')",
"methods": 7,
"unique": 7,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('tooling', 'agent')",
"methods": 6,
"unique": 6,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('springrag', 'mcp')",
"methods": 6,
"unique": 6,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('springrag', 'common')",
"methods": 6,
"unique": 6,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('batch', 'processor')",
"methods": 6,
"unique": 6,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('agent', 'capability')",
"methods": 5,
"unique": 3,
"unique_ratio": 0.6,
"sore": "(listCapabilities.firstOrNull|(AgentExecutionContext|return AgentCapabilityDescriptor(\n id = id,\n capabilityType = capabilityType,\n displayName = describedCapability?.displayName ",
"sore_success": true
},
{
"context": "('test', 'kotlin')",
"methods": 5,
"unique": 5,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('service', 'embedding')",
"methods": 5,
"unique": 5,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('springrag', 'service')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('service', 'agent')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('service', 'document')",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
}
],
"coverage": 1.1,
"elapsed_seconds": 0.01
}
Reference in a new issue View git blame Copy permalink