tobi/grammar-inference-engine
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions
grammar-inference-engine/experiments/results/ragsak_file_path_k1.json

362 lines
8 KiB
JSON
Raw Normal View History

feat: implement Reduce algorithm (Algorithm 4, TODS 2010) - bex/reduce.py: Faithful implementation of Reduce with support-weighted SOA edit distance, adjunction, iterative merging, and Minimize - experiments/context_eval.py: Multi-codebase support (RAGSAK + Flask), Reduce experiments with thresholds 0.05-0.4 - tests/test_reduce.py: 24 tests covering all Reduce components - Flask cloned to external_refs/flask for cross-validation Results: - RAGSAK: 12.0% coverage (First 3 symbols) - Flask: 10.7% coverage (First 3 symbols) - Reduce has minimal impact (1-2 merges per codebase at ε=0.3) - Coverage ceiling appears to be ~10-12% for prefix-based grouping
2026-07-12 00:11:38 +02:00
{
"strategy": "Option A: File path k=1",
"total_contexts": 47,
"meaningful_contexts": 43,
"total_methods": 1594,
"methods_in_good_groups": 17,
"sore_successes": 2,
"sore_failures": 1,
"skip_reasons": {
"too_large": 9,
"too_diverse": 28,
"large_alphabet": 3
},
"groups": [
{
"context": "('controller',)",
"methods": 144,
"unique": 132,
"unique_ratio": 0.917,
"sore": "SKIP(too_large)",
"sore_success": false
},
{
"context": "('job',)",
"methods": 143,
"unique": 133,
"unique_ratio": 0.93,
"sore": "SKIP(too_large)",
"sore_success": false
},
{
"context": "('config',)",
"methods": 140,
"unique": 97,
"unique_ratio": 0.693,
"sore": "SKIP(too_large)",
"sore_success": false
},
{
"context": "('embabel',)",
"methods": 131,
"unique": 119,
"unique_ratio": 0.908,
"sore": "SKIP(too_large)",
"sore_success": false
},
{
"context": "('librechat',)",
"methods": 89,
"unique": 83,
"unique_ratio": 0.933,
"sore": "SKIP(too_large)",
"sore_success": false
},
{
"context": "('architecture',)",
"methods": 87,
"unique": 70,
"unique_ratio": 0.805,
"sore": "SKIP(too_large)",
"sore_success": false
},
{
"context": "('graph',)",
"methods": 81,
"unique": 74,
"unique_ratio": 0.914,
"sore": "SKIP(too_large)",
"sore_success": false
},
{
"context": "('adapter',)",
"methods": 76,
"unique": 74,
"unique_ratio": 0.974,
"sore": "SKIP(too_large)",
"sore_success": false
},
{
"context": "('system',)",
"methods": 58,
"unique": 50,
"unique_ratio": 0.862,
"sore": "SKIP(too_large)",
"sore_success": false
},
{
"context": "('listener',)",
"methods": 50,
"unique": 44,
"unique_ratio": 0.88,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('repository',)",
"methods": 43,
"unique": 39,
"unique_ratio": 0.907,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('storage',)",
"methods": 43,
"unique": 29,
"unique_ratio": 0.674,
"sore": "SKIP(large_alphabet)",
"sore_success": false
},
{
"context": "('support',)",
"methods": 34,
"unique": 32,
"unique_ratio": 0.941,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('batch',)",
"methods": 32,
"unique": 32,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('docling',)",
"methods": 31,
"unique": 28,
"unique_ratio": 0.903,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('service',)",
"methods": 28,
"unique": 25,
"unique_ratio": 0.893,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('knowledgebase',)",
"methods": 27,
"unique": 27,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('ids',)",
"methods": 26,
"unique": 16,
"unique_ratio": 0.615,
"sore": "SKIP(large_alphabet)",
"sore_success": false
},
{
"context": "('chunk',)",
"methods": 25,
"unique": 25,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('health',)",
"methods": 24,
"unique": 22,
"unique_ratio": 0.917,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('simple',)",
"methods": 23,
"unique": 23,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('model',)",
"methods": 23,
"unique": 21,
"unique_ratio": 0.913,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('wikipedia',)",
"methods": 18,
"unique": 17,
"unique_ratio": 0.944,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('chat',)",
"methods": 18,
"unique": 17,
"unique_ratio": 0.944,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('testcontainers',)",
"methods": 18,
"unique": 18,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('steps',)",
"methods": 17,
"unique": 13,
"unique_ratio": 0.765,
"sore": "SKIP(large_alphabet)",
"sore_success": false
},
{
"context": "('writer',)",
"methods": 17,
"unique": 17,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('web',)",
"methods": 16,
"unique": 16,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('reader',)",
"methods": 16,
"unique": 15,
"unique_ratio": 0.938,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('kotlin',)",
"methods": 13,
"unique": 13,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('testsupport',)",
"methods": 12,
"unique": 3,
"unique_ratio": 0.25,
"sore": "(get|(Builder|Any))",
"sore_success": true
},
{
"context": "('recording',)",
"methods": 11,
"unique": 11,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('partition',)",
"methods": 11,
"unique": 11,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('agent',)",
"methods": 9,
"unique": 9,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('cleanup',)",
"methods": 8,
"unique": 6,
"unique_ratio": 0.75,
"sore": "\u2205",
"sore_success": false
},
{
"context": "('rag',)",
"methods": 7,
"unique": 7,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('auth',)",
"methods": 7,
"unique": 7,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('mcp',)",
"methods": 6,
"unique": 6,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('common',)",
"methods": 6,
"unique": 6,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('processor',)",
"methods": 6,
"unique": 6,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('capability',)",
"methods": 5,
"unique": 3,
"unique_ratio": 0.6,
"sore": "(listCapabilities.firstOrNull|(AgentExecutionContext|return AgentCapabilityDescriptor(\n id = id,\n capabilityType = capabilityType,\n displayName = describedCapability?.displayName ",
"sore_success": true
},
{
"context": "('embedding',)",
"methods": 5,
"unique": 5,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
},
{
"context": "('document',)",
"methods": 3,
"unique": 3,
"unique_ratio": 1.0,
"sore": "SKIP(too_diverse)",
"sore_success": false
}
],
"coverage": 1.1,
"elapsed_seconds": 0.0
}
Reference in a new issue Copy permalink