tobi/grammar-inference-engine
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions
grammar-inference-engine/experiments/results/coarsen_ragsak.json
tobjend 592974f039 feat: structural coarsening experiment — keeps function names raw, coarsens keywords 
- coarsen_token() maps tree-sitter captures to categories (RETURN, IF, LOOP, etc.)
- Function calls kept as raw text (they ARE the behavioral content)
- Only ~5% structural tokens coarsened
- Results: Flask coverage 17.5% -> 28.8% (+11.3%), RAGSAK unchanged (95% calls)
- Cross-package shapes emerge: ('IF', 'KW', 'RETURN') in 4 Flask packages
2026-07-12 00:56:09 +02:00

993 lines
No EOL
24 KiB
JSON
Raw Blame History

{
"name": "RAGSAK",
"raw": {
"seq_count": 1594,
"alphabet_size": 1724,
"alphabet_sample": [
"",
"\"Cann",
"\"Fail",
"\"k",
"(",
"(\"",
"()",
"(documen",
"(val scores: List<Double> = emptyList())",
") && s3",
")) {",
",",
".jo",
".va",
"Acti",
"AgentCapabilityDescriptor",
"AgentDecision",
"AgentExecutionContext",
"AndServerWebExchangeMatcher",
"Any",
"AssertionError",
"AssistantMessage",
"AtomicLong",
"AtomicReference",
"AuthController",
"BCryptPasswordEncoder",
"BatchId",
"BatchJobController",
"BatchJobListener",
"BatchNotFoundException"
],
"k1": {
"contexts": 510,
"sore_successes": 39,
"total_methods": 1594,
"methods_in_good": 125,
"coverage": 7.8,
"cross_package_contexts": 145,
"elapsed": 0.001,
"top_groups": [
{
"context": "('every',)",
"label": "raw_k1",
"methods": 102,
"unique": 93,
"unique_ratio": 0.912,
"alphabet_size": 232,
"sore": null,
"sore_success": false
},
{
"context": "('runTest',)",
"label": "raw_k1",
"methods": 59,
"unique": 54,
"unique_ratio": 0.915,
"alphabet_size": 131,
"sore": null,
"sore_success": false
},
{
"context": "('listOf',)",
"label": "raw_k1",
"methods": 37,
"unique": 28,
"unique_ratio": 0.757,
"alphabet_size": 99,
"sore": null,
"sore_success": false
},
{
"context": "('mockk',)",
"label": "raw_k1",
"methods": 31,
"unique": 21,
"unique_ratio": 0.677,
"alphabet_size": 99,
"sore": null,
"sore_success": false
},
{
"context": "('runBlocking',)",
"label": "raw_k1",
"methods": 29,
"unique": 28,
"unique_ratio": 0.966,
"alphabet_size": 115,
"sore": null,
"sore_success": false
},
{
"context": "('trim',)",
"label": "raw_k1",
"methods": 27,
"unique": 24,
"unique_ratio": 0.889,
"alphabet_size": 59,
"sore": null,
"sore_success": false
},
{
"context": "('warn',)",
"label": "raw_k1",
"methods": 26,
"unique": 4,
"unique_ratio": 0.154,
"alphabet_size": 10,
"sore": null,
"sore_success": false
},
{
"context": "('builder',)",
"label": "raw_k1",
"methods": 25,
"unique": 24,
"unique_ratio": 0.96,
"alphabet_size": 128,
"sore": null,
"sore_success": false
},
{
"context": "('info',)",
"label": "raw_k1",
"methods": 23,
"unique": 19,
"unique_ratio": 0.826,
"alphabet_size": 73,
"sore": null,
"sore_success": false
},
{
"context": "('parse',)",
"label": "raw_k1",
"methods": 18,
"unique": 18,
"unique_ratio": 1.0,
"alphabet_size": 49,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('every',)",
"packages": 18
},
{
"context": "('builder',)",
"packages": 16
},
{
"context": "('listOf',)",
"packages": 13
},
{
"context": "('runTest',)",
"packages": 12
},
{
"context": "('info',)",
"packages": 11
},
{
"context": "('mockk',)",
"packages": 11
},
{
"context": "('trim',)",
"packages": 10
},
{
"context": "('isEmpty',)",
"packages": 9
},
{
"context": "('_eps',)",
"packages": 9
},
{
"context": "('debug',)",
"packages": 9
}
]
},
"k2": {
"contexts": 918,
"sore_successes": 76,
"total_methods": 1594,
"methods_in_good": 249,
"coverage": 15.6,
"cross_package_contexts": 100,
"elapsed": 0.001,
"top_groups": [
{
"context": "('listOf', 'listOf')",
"label": "raw_k2",
"methods": 37,
"unique": 28,
"unique_ratio": 0.757,
"alphabet_size": 99,
"sore": null,
"sore_success": false
},
{
"context": "('warn', 'status')",
"label": "raw_k2",
"methods": 22,
"unique": 1,
"unique_ratio": 0.045,
"alphabet_size": 4,
"sore": "warn.status.body.ErrorResponse",
"sore_success": true
},
{
"context": "('_eps',)",
"label": "raw_k2",
"methods": 16,
"unique": 1,
"unique_ratio": 0.062,
"alphabet_size": 0,
"sore": null,
"sore_success": false
},
{
"context": "('VectorChunk', 'mapOf')",
"label": "raw_k2",
"methods": 13,
"unique": 11,
"unique_ratio": 0.846,
"alphabet_size": 42,
"sore": null,
"sore_success": false
},
{
"context": "('ery {', 'stKnowledgeBases()')",
"label": "raw_k2",
"methods": 13,
"unique": 5,
"unique_ratio": 0.385,
"alphabet_size": 13,
"sore": null,
"sore_success": false
},
{
"context": "('productionClasses', 'filter')",
"label": "raw_k2",
"methods": 13,
"unique": 13,
"unique_ratio": 1.0,
"alphabet_size": 16,
"sore": null,
"sore_success": false
},
{
"context": "('mockk',)",
"label": "raw_k2",
"methods": 11,
"unique": 1,
"unique_ratio": 0.091,
"alphabet_size": 1,
"sore": "mockk",
"sore_success": true
},
{
"context": "('filesIn', 'assertTrue')",
"label": "raw_k2",
"methods": 11,
"unique": 5,
"unique_ratio": 0.455,
"alphabet_size": 4,
"sore": null,
"sore_success": false
},
{
"context": "('runTest', 'ChatResponse')",
"label": "raw_k2",
"methods": 10,
"unique": 10,
"unique_ratio": 1.0,
"alphabet_size": 36,
"sore": null,
"sore_success": false
},
{
"context": "('runTest', 'mockk')",
"label": "raw_k2",
"methods": 9,
"unique": 7,
"unique_ratio": 0.778,
"alphabet_size": 27,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('listOf', 'listOf')",
"packages": 13
},
{
"context": "('_eps',)",
"packages": 9
},
{
"context": "('trim', 'lowercase')",
"packages": 5
},
{
"context": "('mutableListOf', 'mutableListOf')",
"packages": 5
},
{
"context": "('defaultCapabilityId',)",
"packages": 4
},
{
"context": "('every', 'defaultCapabilityId')",
"packages": 4
},
{
"context": "('clearAllMocks',)",
"packages": 4
},
{
"context": "('VectorChunk', 'mapOf')",
"packages": 4
},
{
"context": "('Builder',)",
"packages": 4
},
{
"context": "('Any',)",
"packages": 4
}
]
},
"k3": {
"contexts": 1066,
"sore_successes": 89,
"total_methods": 1594,
"methods_in_good": 297,
"coverage": 18.6,
"cross_package_contexts": 69,
"elapsed": 0.0,
"top_groups": [
{
"context": "('warn', 'status', 'body')",
"label": "raw_k3",
"methods": 22,
"unique": 1,
"unique_ratio": 0.045,
"alphabet_size": 4,
"sore": "warn.status.body.ErrorResponse",
"sore_success": true
},
{
"context": "('_eps',)",
"label": "raw_k3",
"methods": 16,
"unique": 1,
"unique_ratio": 0.062,
"alphabet_size": 0,
"sore": null,
"sore_success": false
},
{
"context": "('VectorChunk', 'mapOf', 'mapOf')",
"label": "raw_k3",
"methods": 13,
"unique": 11,
"unique_ratio": 0.846,
"alphabet_size": 42,
"sore": null,
"sore_success": false
},
{
"context": "('ery {', 'stKnowledgeBases()', 'gRequest(m')",
"label": "raw_k3",
"methods": 12,
"unique": 4,
"unique_ratio": 0.333,
"alphabet_size": 11,
"sore": "ery {.stKnowledgeBases().gRequest(m.(ckKnowledgeBase(re.ertEquals(Kn.\"k|eckKnowledgeBase(r.(sertEquals(K.(\"|(sertEquals(t|sertNull(o)))",
"sore_success": true
},
{
"context": "('listOf', 'listOf', 'productionFiles')",
"label": "raw_k3",
"methods": 12,
"unique": 6,
"unique_ratio": 0.5,
"alphabet_size": 9,
"sore": null,
"sore_success": false
},
{
"context": "('mockk',)",
"label": "raw_k3",
"methods": 11,
"unique": 1,
"unique_ratio": 0.091,
"alphabet_size": 1,
"sore": "mockk",
"sore_success": true
},
{
"context": "('File', 'writeBytes', 'byteArrayOf')",
"label": "raw_k3",
"methods": 8,
"unique": 7,
"unique_ratio": 0.875,
"alphabet_size": 25,
"sore": null,
"sore_success": false
},
{
"context": "('runTest', 'ChatResponse', 'emptyList')",
"label": "raw_k3",
"methods": 7,
"unique": 7,
"unique_ratio": 1.0,
"alphabet_size": 33,
"sore": null,
"sore_success": false
},
{
"context": "('filesIn', 'assertTrue', 'hasImport')",
"label": "raw_k3",
"methods": 7,
"unique": 2,
"unique_ratio": 0.286,
"alphabet_size": 3,
"sore": "filesIn.assertTrue.(hasImport)+",
"sore_success": true
},
{
"context": "('assertEquals', 'of', 'assertFailsWith')",
"label": "raw_k3",
"methods": 7,
"unique": 3,
"unique_ratio": 0.429,
"alphabet_size": 3,
"sore": "assertEquals.(of.(assertFailsWith)?)+",
"sore_success": true
}
],
"cross_pkg_examples": [
{
"context": "('_eps',)",
"packages": 9
},
{
"context": "('defaultCapabilityId',)",
"packages": 4
},
{
"context": "('clearAllMocks',)",
"packages": 4
},
{
"context": "('VectorChunk', 'mapOf', 'mapOf')",
"packages": 4
},
{
"context": "('Builder',)",
"packages": 4
},
{
"context": "('Any',)",
"packages": 4
},
{
"context": "('get',)",
"packages": 4
},
{
"context": "('registerProperties',)",
"packages": 4
},
{
"context": "('assumeTrue', 'isDockerAvailable', 'start')",
"packages": 4
},
{
"context": "('listOf', 'listOf', 'forEach')",
"packages": 4
}
]
}
},
"coarsened": {
"seq_count": 1594,
"alphabet_size": 1728,
"alphabet_sample": [
"",
"\"Cann",
"\"Fail",
"\"k",
"(",
"(\"",
"()",
"(documen",
"(val scores: List<Double> = emptyList())",
") && s3",
")) {",
",",
".jo",
".va",
"ATTR",
"Acti",
"AgentCapabilityDescriptor",
"AgentDecision",
"AgentExecutionContext",
"AndServerWebExchangeMatcher",
"Any",
"AssertionError",
"AssistantMessage",
"AtomicLong",
"AtomicReference",
"AuthController",
"BCryptPasswordEncoder",
"BatchId",
"BatchJobController",
"BatchJobListener"
],
"k1": {
"contexts": 474,
"sore_successes": 33,
"total_methods": 1594,
"methods_in_good": 107,
"coverage": 6.7,
"cross_package_contexts": 128,
"elapsed": 0.0,
"top_groups": [
{
"context": "('every',)",
"label": "coarsened_k1",
"methods": 102,
"unique": 93,
"unique_ratio": 0.912,
"alphabet_size": 232,
"sore": null,
"sore_success": false
},
{
"context": "('RETURN',)",
"label": "coarsened_k1",
"methods": 91,
"unique": 75,
"unique_ratio": 0.824,
"alphabet_size": 186,
"sore": null,
"sore_success": false
},
{
"context": "('runTest',)",
"label": "coarsened_k1",
"methods": 59,
"unique": 54,
"unique_ratio": 0.915,
"alphabet_size": 131,
"sore": null,
"sore_success": false
},
{
"context": "('listOf',)",
"label": "coarsened_k1",
"methods": 36,
"unique": 27,
"unique_ratio": 0.75,
"alphabet_size": 95,
"sore": null,
"sore_success": false
},
{
"context": "('mockk',)",
"label": "coarsened_k1",
"methods": 31,
"unique": 21,
"unique_ratio": 0.677,
"alphabet_size": 100,
"sore": null,
"sore_success": false
},
{
"context": "('runBlocking',)",
"label": "coarsened_k1",
"methods": 29,
"unique": 28,
"unique_ratio": 0.966,
"alphabet_size": 117,
"sore": null,
"sore_success": false
},
{
"context": "('trim',)",
"label": "coarsened_k1",
"methods": 26,
"unique": 23,
"unique_ratio": 0.885,
"alphabet_size": 57,
"sore": null,
"sore_success": false
},
{
"context": "('warn',)",
"label": "coarsened_k1",
"methods": 26,
"unique": 4,
"unique_ratio": 0.154,
"alphabet_size": 11,
"sore": null,
"sore_success": false
},
{
"context": "('builder',)",
"label": "coarsened_k1",
"methods": 24,
"unique": 23,
"unique_ratio": 0.958,
"alphabet_size": 126,
"sore": null,
"sore_success": false
},
{
"context": "('info',)",
"label": "coarsened_k1",
"methods": 23,
"unique": 19,
"unique_ratio": 0.826,
"alphabet_size": 74,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('RETURN',)",
"packages": 36
},
{
"context": "('every',)",
"packages": 18
},
{
"context": "('builder',)",
"packages": 15
},
{
"context": "('runTest',)",
"packages": 12
},
{
"context": "('listOf',)",
"packages": 12
},
{
"context": "('info',)",
"packages": 11
},
{
"context": "('mockk',)",
"packages": 11
},
{
"context": "('trim',)",
"packages": 10
},
{
"context": "('isEmpty',)",
"packages": 9
},
{
"context": "('debug',)",
"packages": 9
}
]
},
"k2": {
"contexts": 883,
"sore_successes": 69,
"total_methods": 1594,
"methods_in_good": 225,
"coverage": 14.1,
"cross_package_contexts": 108,
"elapsed": 0.001,
"top_groups": [
{
"context": "('listOf', 'listOf')",
"label": "coarsened_k2",
"methods": 36,
"unique": 27,
"unique_ratio": 0.75,
"alphabet_size": 95,
"sore": null,
"sore_success": false
},
{
"context": "('warn', 'RETURN')",
"label": "coarsened_k2",
"methods": 23,
"unique": 2,
"unique_ratio": 0.087,
"alphabet_size": 7,
"sore": null,
"sore_success": false
},
{
"context": "('RETURN',)",
"label": "coarsened_k2",
"methods": 13,
"unique": 1,
"unique_ratio": 0.077,
"alphabet_size": 1,
"sore": "RETURN",
"sore_success": true
},
{
"context": "('VectorChunk', 'mapOf')",
"label": "coarsened_k2",
"methods": 13,
"unique": 11,
"unique_ratio": 0.846,
"alphabet_size": 42,
"sore": null,
"sore_success": false
},
{
"context": "('ery {', 'stKnowledgeBases()')",
"label": "coarsened_k2",
"methods": 13,
"unique": 5,
"unique_ratio": 0.385,
"alphabet_size": 13,
"sore": null,
"sore_success": false
},
{
"context": "('productionClasses', 'filter')",
"label": "coarsened_k2",
"methods": 13,
"unique": 13,
"unique_ratio": 1.0,
"alphabet_size": 16,
"sore": null,
"sore_success": false
},
{
"context": "('isEmpty', 'RETURN')",
"label": "coarsened_k2",
"methods": 12,
"unique": 12,
"unique_ratio": 1.0,
"alphabet_size": 59,
"sore": null,
"sore_success": false
},
{
"context": "('mockk',)",
"label": "coarsened_k2",
"methods": 11,
"unique": 1,
"unique_ratio": 0.091,
"alphabet_size": 1,
"sore": "mockk",
"sore_success": true
},
{
"context": "('filesIn', 'assertTrue')",
"label": "coarsened_k2",
"methods": 11,
"unique": 5,
"unique_ratio": 0.455,
"alphabet_size": 4,
"sore": null,
"sore_success": false
},
{
"context": "('RETURN', 'RETURN')",
"label": "coarsened_k2",
"methods": 10,
"unique": 10,
"unique_ratio": 1.0,
"alphabet_size": 15,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('listOf', 'listOf')",
"packages": 12
},
{
"context": "('isEmpty', 'RETURN')",
"packages": 9
},
{
"context": "('RETURN',)",
"packages": 9
},
{
"context": "('RETURN', 'RETURN')",
"packages": 7
},
{
"context": "('trim', 'lowercase')",
"packages": 5
},
{
"context": "('mutableListOf', 'mutableListOf')",
"packages": 5
},
{
"context": "('defaultCapabilityId',)",
"packages": 4
},
{
"context": "('every', 'defaultCapabilityId')",
"packages": 4
},
{
"context": "('clearAllMocks',)",
"packages": 4
},
{
"context": "('VectorChunk', 'mapOf')",
"packages": 4
}
]
},
"k3": {
"contexts": 1062,
"sore_successes": 82,
"total_methods": 1594,
"methods_in_good": 292,
"coverage": 18.3,
"cross_package_contexts": 74,
"elapsed": 0.0,
"top_groups": [
{
"context": "('warn', 'RETURN', 'status')",
"label": "coarsened_k3",
"methods": 22,
"unique": 1,
"unique_ratio": 0.045,
"alphabet_size": 5,
"sore": "warn.RETURN.status.body.ErrorResponse",
"sore_success": true
},
{
"context": "('RETURN',)",
"label": "coarsened_k3",
"methods": 13,
"unique": 1,
"unique_ratio": 0.077,
"alphabet_size": 1,
"sore": "RETURN",
"sore_success": true
},
{
"context": "('VectorChunk', 'mapOf', 'mapOf')",
"label": "coarsened_k3",
"methods": 13,
"unique": 11,
"unique_ratio": 0.846,
"alphabet_size": 42,
"sore": null,
"sore_success": false
},
{
"context": "('ery {', 'stKnowledgeBases()', 'gRequest(m')",
"label": "coarsened_k3",
"methods": 12,
"unique": 4,
"unique_ratio": 0.333,
"alphabet_size": 11,
"sore": "ery {.stKnowledgeBases().gRequest(m.(ckKnowledgeBase(re.ertEquals(Kn.\"k|eckKnowledgeBase(r.(sertEquals(K.(\"|(sertEquals(t|sertNull(o)))",
"sore_success": true
},
{
"context": "('listOf', 'listOf', 'productionFiles')",
"label": "coarsened_k3",
"methods": 12,
"unique": 6,
"unique_ratio": 0.5,
"alphabet_size": 9,
"sore": null,
"sore_success": false
},
{
"context": "('mockk',)",
"label": "coarsened_k3",
"methods": 11,
"unique": 1,
"unique_ratio": 0.091,
"alphabet_size": 1,
"sore": "mockk",
"sore_success": true
},
{
"context": "('parseStorageUri', 'RETURN', 'RETURN')",
"label": "coarsened_k3",
"methods": 8,
"unique": 5,
"unique_ratio": 0.625,
"alphabet_size": 14,
"sore": null,
"sore_success": false
},
{
"context": "('File', 'writeBytes', 'byteArrayOf')",
"label": "coarsened_k3",
"methods": 8,
"unique": 7,
"unique_ratio": 0.875,
"alphabet_size": 25,
"sore": null,
"sore_success": false
},
{
"context": "('runTest', 'ChatResponse', 'emptyList')",
"label": "coarsened_k3",
"methods": 7,
"unique": 7,
"unique_ratio": 1.0,
"alphabet_size": 33,
"sore": null,
"sore_success": false
},
{
"context": "('filesIn', 'assertTrue', 'hasImport')",
"label": "coarsened_k3",
"methods": 7,
"unique": 2,
"unique_ratio": 0.286,
"alphabet_size": 3,
"sore": "filesIn.assertTrue.(hasImport)+",
"sore_success": true
}
],
"cross_pkg_examples": [
{
"context": "('RETURN',)",
"packages": 9
},
{
"context": "('defaultCapabilityId',)",
"packages": 4
},
{
"context": "('clearAllMocks',)",
"packages": 4
},
{
"context": "('VectorChunk', 'mapOf', 'mapOf')",
"packages": 4
},
{
"context": "('Builder',)",
"packages": 4
},
{
"context": "('Any',)",
"packages": 4
},
{
"context": "('get',)",
"packages": 4
},
{
"context": "('registerProperties',)",
"packages": 4
},
{
"context": "('assumeTrue', 'isDockerAvailable', 'start')",
"packages": 4
},
{
"context": "('listOf', 'listOf', 'forEach')",
"packages": 4
}
]
}
}
}
Reference in a new issue View git blame Copy permalink