grammar-inference-engine/experiments/results/coarsen_ragsak.json
tobjend d65b78bdf1 fix: bare Kotlin captures + minimal coarsening (RETURN/IF/EXCEPTION/LOOP only)
- Added bare Kotlin captures to BEHAVIORAL_PREFIXES: conditional, exception, repeat, property, type
- Dropped variable (too noisy), kept only 4 high-signal categories in coarsen_token
- RAGSAK: 87 cross-package contexts (up from 69), real null-check patterns
- Flask: k=2 coverage 4.5% -> 15.8%, cross-package rendering conventions
2026-07-12 01:04:51 +02:00

993 lines
No EOL
24 KiB
JSON

{
"name": "RAGSAK",
"raw": {
"seq_count": 1609,
"alphabet_size": 1724,
"alphabet_sample": [
"",
"\"Cann",
"\"Fail",
"\"k",
"(",
"(\"",
"()",
"(documen",
"(val scores: List<Double> = emptyList())",
") && s3",
")) {",
",",
".jo",
".va",
"Acti",
"AgentCapabilityDescriptor",
"AgentDecision",
"AgentExecutionContext",
"AndServerWebExchangeMatcher",
"Any",
"AssertionError",
"AssistantMessage",
"AtomicLong",
"AtomicReference",
"AuthController",
"BCryptPasswordEncoder",
"BatchId",
"BatchJobController",
"BatchJobListener",
"BatchNotFoundException"
],
"k1": {
"contexts": 510,
"sore_successes": 39,
"total_methods": 1609,
"methods_in_good": 125,
"coverage": 7.8,
"cross_package_contexts": 145,
"elapsed": 0.001,
"top_groups": [
{
"context": "('every',)",
"label": "raw_k1",
"methods": 102,
"unique": 93,
"unique_ratio": 0.912,
"alphabet_size": 232,
"sore": null,
"sore_success": false
},
{
"context": "('runTest',)",
"label": "raw_k1",
"methods": 59,
"unique": 54,
"unique_ratio": 0.915,
"alphabet_size": 131,
"sore": null,
"sore_success": false
},
{
"context": "('listOf',)",
"label": "raw_k1",
"methods": 37,
"unique": 28,
"unique_ratio": 0.757,
"alphabet_size": 99,
"sore": null,
"sore_success": false
},
{
"context": "('_eps',)",
"label": "raw_k1",
"methods": 31,
"unique": 1,
"unique_ratio": 0.032,
"alphabet_size": 0,
"sore": null,
"sore_success": false
},
{
"context": "('mockk',)",
"label": "raw_k1",
"methods": 31,
"unique": 21,
"unique_ratio": 0.677,
"alphabet_size": 99,
"sore": null,
"sore_success": false
},
{
"context": "('runBlocking',)",
"label": "raw_k1",
"methods": 29,
"unique": 28,
"unique_ratio": 0.966,
"alphabet_size": 115,
"sore": null,
"sore_success": false
},
{
"context": "('trim',)",
"label": "raw_k1",
"methods": 27,
"unique": 24,
"unique_ratio": 0.889,
"alphabet_size": 59,
"sore": null,
"sore_success": false
},
{
"context": "('warn',)",
"label": "raw_k1",
"methods": 26,
"unique": 4,
"unique_ratio": 0.154,
"alphabet_size": 10,
"sore": null,
"sore_success": false
},
{
"context": "('builder',)",
"label": "raw_k1",
"methods": 25,
"unique": 24,
"unique_ratio": 0.96,
"alphabet_size": 128,
"sore": null,
"sore_success": false
},
{
"context": "('info',)",
"label": "raw_k1",
"methods": 23,
"unique": 19,
"unique_ratio": 0.826,
"alphabet_size": 73,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('every',)",
"packages": 18
},
{
"context": "('_eps',)",
"packages": 18
},
{
"context": "('builder',)",
"packages": 16
},
{
"context": "('listOf',)",
"packages": 13
},
{
"context": "('runTest',)",
"packages": 12
},
{
"context": "('info',)",
"packages": 11
},
{
"context": "('mockk',)",
"packages": 11
},
{
"context": "('trim',)",
"packages": 10
},
{
"context": "('isEmpty',)",
"packages": 9
},
{
"context": "('debug',)",
"packages": 9
}
]
},
"k2": {
"contexts": 918,
"sore_successes": 76,
"total_methods": 1609,
"methods_in_good": 249,
"coverage": 15.5,
"cross_package_contexts": 100,
"elapsed": 0.001,
"top_groups": [
{
"context": "('listOf', 'listOf')",
"label": "raw_k2",
"methods": 37,
"unique": 28,
"unique_ratio": 0.757,
"alphabet_size": 99,
"sore": null,
"sore_success": false
},
{
"context": "('_eps',)",
"label": "raw_k2",
"methods": 31,
"unique": 1,
"unique_ratio": 0.032,
"alphabet_size": 0,
"sore": null,
"sore_success": false
},
{
"context": "('warn', 'status')",
"label": "raw_k2",
"methods": 22,
"unique": 1,
"unique_ratio": 0.045,
"alphabet_size": 4,
"sore": "warn.status.body.ErrorResponse",
"sore_success": true
},
{
"context": "('VectorChunk', 'mapOf')",
"label": "raw_k2",
"methods": 13,
"unique": 11,
"unique_ratio": 0.846,
"alphabet_size": 42,
"sore": null,
"sore_success": false
},
{
"context": "('ery {', 'stKnowledgeBases()')",
"label": "raw_k2",
"methods": 13,
"unique": 5,
"unique_ratio": 0.385,
"alphabet_size": 13,
"sore": null,
"sore_success": false
},
{
"context": "('productionClasses', 'filter')",
"label": "raw_k2",
"methods": 13,
"unique": 13,
"unique_ratio": 1.0,
"alphabet_size": 16,
"sore": null,
"sore_success": false
},
{
"context": "('mockk',)",
"label": "raw_k2",
"methods": 11,
"unique": 1,
"unique_ratio": 0.091,
"alphabet_size": 1,
"sore": "mockk",
"sore_success": true
},
{
"context": "('filesIn', 'assertTrue')",
"label": "raw_k2",
"methods": 11,
"unique": 5,
"unique_ratio": 0.455,
"alphabet_size": 4,
"sore": null,
"sore_success": false
},
{
"context": "('runTest', 'ChatResponse')",
"label": "raw_k2",
"methods": 10,
"unique": 10,
"unique_ratio": 1.0,
"alphabet_size": 36,
"sore": null,
"sore_success": false
},
{
"context": "('runTest', 'mockk')",
"label": "raw_k2",
"methods": 9,
"unique": 7,
"unique_ratio": 0.778,
"alphabet_size": 27,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('_eps',)",
"packages": 18
},
{
"context": "('listOf', 'listOf')",
"packages": 13
},
{
"context": "('trim', 'lowercase')",
"packages": 5
},
{
"context": "('mutableListOf', 'mutableListOf')",
"packages": 5
},
{
"context": "('defaultCapabilityId',)",
"packages": 4
},
{
"context": "('every', 'defaultCapabilityId')",
"packages": 4
},
{
"context": "('clearAllMocks',)",
"packages": 4
},
{
"context": "('VectorChunk', 'mapOf')",
"packages": 4
},
{
"context": "('Builder',)",
"packages": 4
},
{
"context": "('Any',)",
"packages": 4
}
]
},
"k3": {
"contexts": 1066,
"sore_successes": 89,
"total_methods": 1609,
"methods_in_good": 297,
"coverage": 18.5,
"cross_package_contexts": 69,
"elapsed": 0.001,
"top_groups": [
{
"context": "('_eps',)",
"label": "raw_k3",
"methods": 31,
"unique": 1,
"unique_ratio": 0.032,
"alphabet_size": 0,
"sore": null,
"sore_success": false
},
{
"context": "('warn', 'status', 'body')",
"label": "raw_k3",
"methods": 22,
"unique": 1,
"unique_ratio": 0.045,
"alphabet_size": 4,
"sore": "warn.status.body.ErrorResponse",
"sore_success": true
},
{
"context": "('VectorChunk', 'mapOf', 'mapOf')",
"label": "raw_k3",
"methods": 13,
"unique": 11,
"unique_ratio": 0.846,
"alphabet_size": 42,
"sore": null,
"sore_success": false
},
{
"context": "('ery {', 'stKnowledgeBases()', 'gRequest(m')",
"label": "raw_k3",
"methods": 12,
"unique": 4,
"unique_ratio": 0.333,
"alphabet_size": 11,
"sore": "ery {.stKnowledgeBases().gRequest(m.(ckKnowledgeBase(re.ertEquals(Kn.\"k|eckKnowledgeBase(r.(sertEquals(K.(\"|(sertEquals(t|sertNull(o)))",
"sore_success": true
},
{
"context": "('listOf', 'listOf', 'productionFiles')",
"label": "raw_k3",
"methods": 12,
"unique": 6,
"unique_ratio": 0.5,
"alphabet_size": 9,
"sore": null,
"sore_success": false
},
{
"context": "('mockk',)",
"label": "raw_k3",
"methods": 11,
"unique": 1,
"unique_ratio": 0.091,
"alphabet_size": 1,
"sore": "mockk",
"sore_success": true
},
{
"context": "('File', 'writeBytes', 'byteArrayOf')",
"label": "raw_k3",
"methods": 8,
"unique": 7,
"unique_ratio": 0.875,
"alphabet_size": 25,
"sore": null,
"sore_success": false
},
{
"context": "('runTest', 'ChatResponse', 'emptyList')",
"label": "raw_k3",
"methods": 7,
"unique": 7,
"unique_ratio": 1.0,
"alphabet_size": 33,
"sore": null,
"sore_success": false
},
{
"context": "('filesIn', 'assertTrue', 'hasImport')",
"label": "raw_k3",
"methods": 7,
"unique": 2,
"unique_ratio": 0.286,
"alphabet_size": 3,
"sore": "filesIn.assertTrue.(hasImport)+",
"sore_success": true
},
{
"context": "('of',)",
"label": "raw_k3",
"methods": 7,
"unique": 1,
"unique_ratio": 0.143,
"alphabet_size": 1,
"sore": "of",
"sore_success": true
}
],
"cross_pkg_examples": [
{
"context": "('_eps',)",
"packages": 18
},
{
"context": "('defaultCapabilityId',)",
"packages": 4
},
{
"context": "('clearAllMocks',)",
"packages": 4
},
{
"context": "('VectorChunk', 'mapOf', 'mapOf')",
"packages": 4
},
{
"context": "('Builder',)",
"packages": 4
},
{
"context": "('Any',)",
"packages": 4
},
{
"context": "('get',)",
"packages": 4
},
{
"context": "('registerProperties',)",
"packages": 4
},
{
"context": "('assumeTrue', 'isDockerAvailable', 'start')",
"packages": 4
},
{
"context": "('listOf', 'listOf', 'forEach')",
"packages": 4
}
]
}
},
"coarsened": {
"seq_count": 1609,
"alphabet_size": 2468,
"alphabet_sample": [
"",
"\" }",
"\"Cann",
"\"Fail",
"\"k",
"\"una",
"(",
"(\"",
"()",
"(documen",
"(val scores: List<Double> = emptyList())",
")",
") && s3",
") return",
")) {",
",",
".con",
".end",
".jo",
".lin",
".mapInde",
".size).t",
".va",
": 0",
"==",
"> 5",
"?.na",
"?:",
"@",
"ABANDONED"
],
"k1": {
"contexts": 476,
"sore_successes": 16,
"total_methods": 1609,
"methods_in_good": 51,
"coverage": 3.2,
"cross_package_contexts": 123,
"elapsed": 0.0,
"top_groups": [
{
"context": "('IF',)",
"label": "coarsened_k1",
"methods": 104,
"unique": 98,
"unique_ratio": 0.942,
"alphabet_size": 463,
"sore": null,
"sore_success": false
},
{
"context": "('every',)",
"label": "coarsened_k1",
"methods": 102,
"unique": 95,
"unique_ratio": 0.931,
"alphabet_size": 304,
"sore": null,
"sore_success": false
},
{
"context": "('RETURN',)",
"label": "coarsened_k1",
"methods": 61,
"unique": 51,
"unique_ratio": 0.836,
"alphabet_size": 210,
"sore": null,
"sore_success": false
},
{
"context": "('runTest',)",
"label": "coarsened_k1",
"methods": 59,
"unique": 54,
"unique_ratio": 0.915,
"alphabet_size": 171,
"sore": null,
"sore_success": false
},
{
"context": "('listOf',)",
"label": "coarsened_k1",
"methods": 33,
"unique": 25,
"unique_ratio": 0.758,
"alphabet_size": 111,
"sore": null,
"sore_success": false
},
{
"context": "('mockk',)",
"label": "coarsened_k1",
"methods": 30,
"unique": 20,
"unique_ratio": 0.667,
"alphabet_size": 129,
"sore": null,
"sore_success": false
},
{
"context": "('runBlocking',)",
"label": "coarsened_k1",
"methods": 29,
"unique": 28,
"unique_ratio": 0.966,
"alphabet_size": 165,
"sore": null,
"sore_success": false
},
{
"context": "('builder',)",
"label": "coarsened_k1",
"methods": 23,
"unique": 23,
"unique_ratio": 1.0,
"alphabet_size": 164,
"sore": null,
"sore_success": false
},
{
"context": "('warn',)",
"label": "coarsened_k1",
"methods": 23,
"unique": 14,
"unique_ratio": 0.609,
"alphabet_size": 23,
"sore": null,
"sore_success": false
},
{
"context": "('info',)",
"label": "coarsened_k1",
"methods": 19,
"unique": 15,
"unique_ratio": 0.789,
"alphabet_size": 88,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('IF',)",
"packages": 34
},
{
"context": "('RETURN',)",
"packages": 26
},
{
"context": "('every',)",
"packages": 18
},
{
"context": "('builder',)",
"packages": 15
},
{
"context": "('runTest',)",
"packages": 12
},
{
"context": "('mockk',)",
"packages": 11
},
{
"context": "('listOf',)",
"packages": 11
},
{
"context": "('info',)",
"packages": 10
},
{
"context": "('get',)",
"packages": 8
},
{
"context": "('debug',)",
"packages": 8
}
]
},
"k2": {
"contexts": 782,
"sore_successes": 25,
"total_methods": 1609,
"methods_in_good": 82,
"coverage": 5.1,
"cross_package_contexts": 123,
"elapsed": 0.001,
"top_groups": [
{
"context": "('listOf', 'listOf')",
"label": "coarsened_k2",
"methods": 33,
"unique": 25,
"unique_ratio": 0.758,
"alphabet_size": 111,
"sore": null,
"sore_success": false
},
{
"context": "('builder', 'builder')",
"label": "coarsened_k2",
"methods": 23,
"unique": 23,
"unique_ratio": 1.0,
"alphabet_size": 164,
"sore": null,
"sore_success": false
},
{
"context": "('warn', 'warn')",
"label": "coarsened_k2",
"methods": 23,
"unique": 14,
"unique_ratio": 0.609,
"alphabet_size": 23,
"sore": null,
"sore_success": false
},
{
"context": "('info', 'info')",
"label": "coarsened_k2",
"methods": 19,
"unique": 15,
"unique_ratio": 0.789,
"alphabet_size": 88,
"sore": null,
"sore_success": false
},
{
"context": "('parse', 'parse')",
"label": "coarsened_k2",
"methods": 18,
"unique": 18,
"unique_ratio": 1.0,
"alphabet_size": 70,
"sore": null,
"sore_success": false
},
{
"context": "('trim', 'trim')",
"label": "coarsened_k2",
"methods": 15,
"unique": 15,
"unique_ratio": 1.0,
"alphabet_size": 54,
"sore": null,
"sore_success": false
},
{
"context": "('VectorChunk', 'mapOf')",
"label": "coarsened_k2",
"methods": 13,
"unique": 11,
"unique_ratio": 0.846,
"alphabet_size": 57,
"sore": null,
"sore_success": false
},
{
"context": "('ery {', 'stKnowledgeBases()')",
"label": "coarsened_k2",
"methods": 13,
"unique": 5,
"unique_ratio": 0.385,
"alphabet_size": 19,
"sore": null,
"sore_success": false
},
{
"context": "('of', 'of')",
"label": "coarsened_k2",
"methods": 13,
"unique": 7,
"unique_ratio": 0.538,
"alphabet_size": 40,
"sore": null,
"sore_success": false
},
{
"context": "('productionClasses', 'filter')",
"label": "coarsened_k2",
"methods": 13,
"unique": 13,
"unique_ratio": 1.0,
"alphabet_size": 28,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('builder', 'builder')",
"packages": 15
},
{
"context": "('listOf', 'listOf')",
"packages": 11
},
{
"context": "('info', 'info')",
"packages": 10
},
{
"context": "('IF', 'isEmpty')",
"packages": 8
},
{
"context": "('IF', 'RETURN')",
"packages": 8
},
{
"context": "('get', 'get')",
"packages": 8
},
{
"context": "('debug', 'debug')",
"packages": 8
},
{
"context": "('trim', 'trim')",
"packages": 7
},
{
"context": "('IF', 'isNullOrBlank')",
"packages": 6
},
{
"context": "('run', 'run')",
"packages": 6
}
]
},
"k3": {
"contexts": 992,
"sore_successes": 29,
"total_methods": 1609,
"methods_in_good": 97,
"coverage": 6.0,
"cross_package_contexts": 87,
"elapsed": 0.001,
"top_groups": [
{
"context": "('warn', 'warn', 'message')",
"label": "coarsened_k3",
"methods": 20,
"unique": 11,
"unique_ratio": 0.55,
"alphabet_size": 13,
"sore": null,
"sore_success": false
},
{
"context": "('VectorChunk', 'mapOf', 'mapOf')",
"label": "coarsened_k3",
"methods": 13,
"unique": 11,
"unique_ratio": 0.846,
"alphabet_size": 57,
"sore": null,
"sore_success": false
},
{
"context": "('ery {', 'stKnowledgeBases()', 'stKnowledgeBases()')",
"label": "coarsened_k3",
"methods": 13,
"unique": 5,
"unique_ratio": 0.385,
"alphabet_size": 19,
"sore": null,
"sore_success": false
},
{
"context": "('productionClasses', 'filter', 'filter')",
"label": "coarsened_k3",
"methods": 13,
"unique": 13,
"unique_ratio": 1.0,
"alphabet_size": 28,
"sore": null,
"sore_success": false
},
{
"context": "('listOf', 'listOf', 'productionFiles')",
"label": "coarsened_k3",
"methods": 12,
"unique": 6,
"unique_ratio": 0.5,
"alphabet_size": 13,
"sore": null,
"sore_success": false
},
{
"context": "('IF', 'isEmpty', 'isEmpty')",
"label": "coarsened_k3",
"methods": 11,
"unique": 11,
"unique_ratio": 1.0,
"alphabet_size": 69,
"sore": null,
"sore_success": false
},
{
"context": "('mockk',)",
"label": "coarsened_k3",
"methods": 11,
"unique": 1,
"unique_ratio": 0.091,
"alphabet_size": 1,
"sore": "mockk",
"sore_success": true
},
{
"context": "('filesIn', 'assertTrue', 'assertTrue')",
"label": "coarsened_k3",
"methods": 11,
"unique": 5,
"unique_ratio": 0.455,
"alphabet_size": 5,
"sore": null,
"sore_success": false
},
{
"context": "('IF', 'isNullOrBlank', 'isNullOrBlank')",
"label": "coarsened_k3",
"methods": 9,
"unique": 7,
"unique_ratio": 0.778,
"alphabet_size": 49,
"sore": null,
"sore_success": false
},
{
"context": "('parseStorageUri', 'value', 'RETURN')",
"label": "coarsened_k3",
"methods": 9,
"unique": 6,
"unique_ratio": 0.667,
"alphabet_size": 20,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('IF', 'isEmpty', 'isEmpty')",
"packages": 8
},
{
"context": "('IF', 'isNullOrBlank', 'isNullOrBlank')",
"packages": 6
},
{
"context": "('IF', 'isBlank', 'isBlank')",
"packages": 5
},
{
"context": "('every', 'defaultCapabilityId', 'defaultCapabilityId')",
"packages": 4
},
{
"context": "('clearAllMocks',)",
"packages": 4
},
{
"context": "('VectorChunk', 'mapOf', 'mapOf')",
"packages": 4
},
{
"context": "('Builder',)",
"packages": 4
},
{
"context": "('Any',)",
"packages": 4
},
{
"context": "('get', 'get')",
"packages": 4
},
{
"context": "('registerProperties', 'registerProperties')",
"packages": 4
}
]
}
}
}