bbdfe93679
Key findings: - File path grouping (structural) = baseline (0.6% coverage) — no improvement - First-k-symbols grouping (behavioral) = 20× improvement (12% coverage at k=3) - Two-dimensional (path + symbols) = worse than behavioral alone - Winner: Option B k=3 — 47 SOREs, 12% coverage Smart filters make experiments fast (<1s vs minutes): - max_unique_ratio=0.85 - max_alphabet=20 - max_soa_edges=100 Preserved in experiments/results/ for future reference.
362 lines
No EOL
8 KiB
JSON
362 lines
No EOL
8 KiB
JSON
{
|
|
"strategy": "Option A: File path k=1",
|
|
"total_contexts": 47,
|
|
"meaningful_contexts": 43,
|
|
"total_methods": 1594,
|
|
"methods_in_good_groups": 17,
|
|
"sore_successes": 2,
|
|
"sore_failures": 1,
|
|
"skip_reasons": {
|
|
"too_large": 9,
|
|
"too_diverse": 28,
|
|
"large_alphabet": 3
|
|
},
|
|
"groups": [
|
|
{
|
|
"context": "('controller',)",
|
|
"methods": 144,
|
|
"unique": 132,
|
|
"unique_ratio": 0.917,
|
|
"sore": "SKIP(too_large)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('job',)",
|
|
"methods": 143,
|
|
"unique": 133,
|
|
"unique_ratio": 0.93,
|
|
"sore": "SKIP(too_large)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('config',)",
|
|
"methods": 140,
|
|
"unique": 97,
|
|
"unique_ratio": 0.693,
|
|
"sore": "SKIP(too_large)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('embabel',)",
|
|
"methods": 131,
|
|
"unique": 119,
|
|
"unique_ratio": 0.908,
|
|
"sore": "SKIP(too_large)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('librechat',)",
|
|
"methods": 89,
|
|
"unique": 83,
|
|
"unique_ratio": 0.933,
|
|
"sore": "SKIP(too_large)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('architecture',)",
|
|
"methods": 87,
|
|
"unique": 70,
|
|
"unique_ratio": 0.805,
|
|
"sore": "SKIP(too_large)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('graph',)",
|
|
"methods": 81,
|
|
"unique": 74,
|
|
"unique_ratio": 0.914,
|
|
"sore": "SKIP(too_large)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('adapter',)",
|
|
"methods": 76,
|
|
"unique": 74,
|
|
"unique_ratio": 0.974,
|
|
"sore": "SKIP(too_large)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('system',)",
|
|
"methods": 58,
|
|
"unique": 50,
|
|
"unique_ratio": 0.862,
|
|
"sore": "SKIP(too_large)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('listener',)",
|
|
"methods": 50,
|
|
"unique": 44,
|
|
"unique_ratio": 0.88,
|
|
"sore": "SKIP(too_diverse)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('repository',)",
|
|
"methods": 43,
|
|
"unique": 39,
|
|
"unique_ratio": 0.907,
|
|
"sore": "SKIP(too_diverse)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('storage',)",
|
|
"methods": 43,
|
|
"unique": 29,
|
|
"unique_ratio": 0.674,
|
|
"sore": "SKIP(large_alphabet)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('support',)",
|
|
"methods": 34,
|
|
"unique": 32,
|
|
"unique_ratio": 0.941,
|
|
"sore": "SKIP(too_diverse)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('batch',)",
|
|
"methods": 32,
|
|
"unique": 32,
|
|
"unique_ratio": 1.0,
|
|
"sore": "SKIP(too_diverse)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('docling',)",
|
|
"methods": 31,
|
|
"unique": 28,
|
|
"unique_ratio": 0.903,
|
|
"sore": "SKIP(too_diverse)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('service',)",
|
|
"methods": 28,
|
|
"unique": 25,
|
|
"unique_ratio": 0.893,
|
|
"sore": "SKIP(too_diverse)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('knowledgebase',)",
|
|
"methods": 27,
|
|
"unique": 27,
|
|
"unique_ratio": 1.0,
|
|
"sore": "SKIP(too_diverse)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('ids',)",
|
|
"methods": 26,
|
|
"unique": 16,
|
|
"unique_ratio": 0.615,
|
|
"sore": "SKIP(large_alphabet)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('chunk',)",
|
|
"methods": 25,
|
|
"unique": 25,
|
|
"unique_ratio": 1.0,
|
|
"sore": "SKIP(too_diverse)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('health',)",
|
|
"methods": 24,
|
|
"unique": 22,
|
|
"unique_ratio": 0.917,
|
|
"sore": "SKIP(too_diverse)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('simple',)",
|
|
"methods": 23,
|
|
"unique": 23,
|
|
"unique_ratio": 1.0,
|
|
"sore": "SKIP(too_diverse)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('model',)",
|
|
"methods": 23,
|
|
"unique": 21,
|
|
"unique_ratio": 0.913,
|
|
"sore": "SKIP(too_diverse)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('wikipedia',)",
|
|
"methods": 18,
|
|
"unique": 17,
|
|
"unique_ratio": 0.944,
|
|
"sore": "SKIP(too_diverse)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('chat',)",
|
|
"methods": 18,
|
|
"unique": 17,
|
|
"unique_ratio": 0.944,
|
|
"sore": "SKIP(too_diverse)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('testcontainers',)",
|
|
"methods": 18,
|
|
"unique": 18,
|
|
"unique_ratio": 1.0,
|
|
"sore": "SKIP(too_diverse)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('steps',)",
|
|
"methods": 17,
|
|
"unique": 13,
|
|
"unique_ratio": 0.765,
|
|
"sore": "SKIP(large_alphabet)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('writer',)",
|
|
"methods": 17,
|
|
"unique": 17,
|
|
"unique_ratio": 1.0,
|
|
"sore": "SKIP(too_diverse)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('web',)",
|
|
"methods": 16,
|
|
"unique": 16,
|
|
"unique_ratio": 1.0,
|
|
"sore": "SKIP(too_diverse)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('reader',)",
|
|
"methods": 16,
|
|
"unique": 15,
|
|
"unique_ratio": 0.938,
|
|
"sore": "SKIP(too_diverse)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('kotlin',)",
|
|
"methods": 13,
|
|
"unique": 13,
|
|
"unique_ratio": 1.0,
|
|
"sore": "SKIP(too_diverse)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('testsupport',)",
|
|
"methods": 12,
|
|
"unique": 3,
|
|
"unique_ratio": 0.25,
|
|
"sore": "(get|(Builder|Any))",
|
|
"sore_success": true
|
|
},
|
|
{
|
|
"context": "('recording',)",
|
|
"methods": 11,
|
|
"unique": 11,
|
|
"unique_ratio": 1.0,
|
|
"sore": "SKIP(too_diverse)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('partition',)",
|
|
"methods": 11,
|
|
"unique": 11,
|
|
"unique_ratio": 1.0,
|
|
"sore": "SKIP(too_diverse)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('agent',)",
|
|
"methods": 9,
|
|
"unique": 9,
|
|
"unique_ratio": 1.0,
|
|
"sore": "SKIP(too_diverse)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('cleanup',)",
|
|
"methods": 8,
|
|
"unique": 6,
|
|
"unique_ratio": 0.75,
|
|
"sore": "\u2205",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('rag',)",
|
|
"methods": 7,
|
|
"unique": 7,
|
|
"unique_ratio": 1.0,
|
|
"sore": "SKIP(too_diverse)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('auth',)",
|
|
"methods": 7,
|
|
"unique": 7,
|
|
"unique_ratio": 1.0,
|
|
"sore": "SKIP(too_diverse)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('mcp',)",
|
|
"methods": 6,
|
|
"unique": 6,
|
|
"unique_ratio": 1.0,
|
|
"sore": "SKIP(too_diverse)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('common',)",
|
|
"methods": 6,
|
|
"unique": 6,
|
|
"unique_ratio": 1.0,
|
|
"sore": "SKIP(too_diverse)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('processor',)",
|
|
"methods": 6,
|
|
"unique": 6,
|
|
"unique_ratio": 1.0,
|
|
"sore": "SKIP(too_diverse)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('capability',)",
|
|
"methods": 5,
|
|
"unique": 3,
|
|
"unique_ratio": 0.6,
|
|
"sore": "(return AgentCapabilityDescriptor(\n id = id,\n capabilityType = capabilityType,\n displayName = describedCapability?.displayName ?: id,\n description = describedCapability?.de",
|
|
"sore_success": true
|
|
},
|
|
{
|
|
"context": "('embedding',)",
|
|
"methods": 5,
|
|
"unique": 5,
|
|
"unique_ratio": 1.0,
|
|
"sore": "SKIP(too_diverse)",
|
|
"sore_success": false
|
|
},
|
|
{
|
|
"context": "('document',)",
|
|
"methods": 3,
|
|
"unique": 3,
|
|
"unique_ratio": 1.0,
|
|
"sore": "SKIP(too_diverse)",
|
|
"sore_success": false
|
|
}
|
|
],
|
|
"coverage": 1.1,
|
|
"elapsed_seconds": 0.01
|
|
} |