tobi/grammar-inference-engine
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions
grammar-inference-engine/experiments/results/coarsen_coroutines.json

993 lines
24 KiB
JSON
Raw Normal View History

feat: 4-codebase evaluation — conventions vs completions tradeoff Added kotlinx.coroutines (1040 .kt files) and FastAPI (1129 .py files). Key findings across 4 codebases: - Coarsening trades per-package coverage for cross-package reach - FastAPI: coverage IMPROVES (12.4% -> 20.7%), cross-pkg +15 - Coroutines: cross-pkg +15, real exception-handling patterns - RAGSAK: cross-pkg +18, null-check conventions across 8 packages - Flask: slight cross-pkg loss, rendering conventions still visible The conventions vs completions tradeoff is real and measurable.
2026-07-12 01:14:09 +02:00
{
"name": "Kotlin Coroutines",
"raw": {
"seq_count": 6720,
"alphabet_size": 2962,
"alphabet_sample": [
"",
"(\n val data: String\n )",
"(\"un",
"()",
"() }",
"() } }",
"())",
"()) }",
"(),",
"(), 1.1)",
"(), self",
"().availab",
"(0)",
"(0),",
"(0, 100)",
"(0, 1024",
"(1)",
"(2)",
"(3)",
"(Bal",
"(Sta",
"(Sto",
"(Thread",
"(Uni",
"(actorSta",
"(block: Runnable)",
"(cou",
"(data: String)",
"(e: Throwable",
"(initial: T)"
],
"k1": {
"contexts": 1128,
"sore_successes": 217,
"total_methods": 6720,
"methods_in_good": 1048,
"coverage": 15.6,
"cross_package_contexts": 347,
"elapsed": 0.009,
"top_groups": [
{
"context": "('runTest',)",
"label": "raw_k1",
"methods": 1693,
"unique": 1483,
"unique_ratio": 0.876,
"alphabet_size": 747,
"sore": null,
"sore_success": false
},
{
"context": "('runBlocking',)",
"label": "raw_k1",
"methods": 367,
"unique": 281,
"unique_ratio": 0.766,
"alphabet_size": 373,
"sore": null,
"sore_success": false
},
{
"context": "('_eps',)",
"label": "raw_k1",
"methods": 161,
"unique": 1,
"unique_ratio": 0.006,
"alphabet_size": 0,
"sore": null,
"sore_success": false
},
{
"context": "('expect',)",
"label": "raw_k1",
"methods": 120,
"unique": 42,
"unique_ratio": 0.35,
"alphabet_size": 69,
"sore": null,
"sore_success": false
},
{
"context": "('test',)",
"label": "raw_k1",
"methods": 88,
"unique": 10,
"unique_ratio": 0.114,
"alphabet_size": 16,
"sore": null,
"sore_success": false
},
{
"context": "('expectUnreached',)",
"label": "raw_k1",
"methods": 76,
"unique": 1,
"unique_ratio": 0.013,
"alphabet_size": 1,
"sore": "expectUnreached",
"sore_success": true
},
{
"context": "('flow',)",
"label": "raw_k1",
"methods": 58,
"unique": 34,
"unique_ratio": 0.586,
"alphabet_size": 79,
"sore": null,
"sore_success": false
},
{
"context": "('launch',)",
"label": "raw_k1",
"methods": 54,
"unique": 34,
"unique_ratio": 0.63,
"alphabet_size": 94,
"sore": null,
"sore_success": false
},
{
"context": "('close',)",
"label": "raw_k1",
"methods": 51,
"unique": 9,
"unique_ratio": 0.176,
"alphabet_size": 11,
"sore": null,
"sore_success": false
},
{
"context": "('rxObservable',)",
"label": "raw_k1",
"methods": 47,
"unique": 22,
"unique_ratio": 0.468,
"alphabet_size": 34,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('_eps',)",
"packages": 48
},
{
"context": "('runTest',)",
"packages": 46
},
{
"context": "('runBlocking',)",
"packages": 41
},
{
"context": "('block',)",
"packages": 31
},
{
"context": "('close',)",
"packages": 20
},
{
"context": "('suspendCancellableCoroutine',)",
"packages": 20
},
{
"context": "('error',)",
"packages": 18
},
{
"context": "('cancel',)",
"packages": 17
},
{
"context": "('launch',)",
"packages": 16
},
{
"context": "('require',)",
"packages": 15
}
]
},
"k2": {
"contexts": 2455,
"sore_successes": 431,
"total_methods": 6720,
"methods_in_good": 1704,
"coverage": 25.4,
"cross_package_contexts": 461,
"elapsed": 0.007,
"top_groups": [
{
"context": "('runTest', 'expect')",
"label": "raw_k2",
"methods": 397,
"unique": 331,
"unique_ratio": 0.834,
"alphabet_size": 272,
"sore": null,
"sore_success": false
},
{
"context": "('_eps',)",
"label": "raw_k2",
"methods": 161,
"unique": 1,
"unique_ratio": 0.006,
"alphabet_size": 0,
"sore": null,
"sore_success": false
},
{
"context": "('runTest', 'flow')",
"label": "raw_k2",
"methods": 143,
"unique": 140,
"unique_ratio": 0.979,
"alphabet_size": 103,
"sore": null,
"sore_success": false
},
{
"context": "('runTest', 'Channel')",
"label": "raw_k2",
"methods": 120,
"unique": 109,
"unique_ratio": 0.908,
"alphabet_size": 107,
"sore": null,
"sore_success": false
},
{
"context": "('test', 'main')",
"label": "raw_k2",
"methods": 88,
"unique": 10,
"unique_ratio": 0.114,
"alphabet_size": 16,
"sore": null,
"sore_success": false
},
{
"context": "('runTest', 'flowOf')",
"label": "raw_k2",
"methods": 80,
"unique": 74,
"unique_ratio": 0.925,
"alphabet_size": 102,
"sore": null,
"sore_success": false
},
{
"context": "('runBlocking', 'expect')",
"label": "raw_k2",
"methods": 76,
"unique": 55,
"unique_ratio": 0.724,
"alphabet_size": 75,
"sore": null,
"sore_success": false
},
{
"context": "('expectUnreached',)",
"label": "raw_k2",
"methods": 76,
"unique": 1,
"unique_ratio": 0.013,
"alphabet_size": 1,
"sore": "expectUnreached",
"sore_success": true
},
{
"context": "('runTest', 'launch')",
"label": "raw_k2",
"methods": 50,
"unique": 47,
"unique_ratio": 0.94,
"alphabet_size": 73,
"sore": null,
"sore_success": false
},
{
"context": "('check', 'check')",
"label": "raw_k2",
"methods": 45,
"unique": 40,
"unique_ratio": 0.889,
"alphabet_size": 87,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('_eps',)",
"packages": 48
},
{
"context": "('runTest', 'expect')",
"packages": 26
},
{
"context": "('error', 'error')",
"packages": 18
},
{
"context": "('close',)",
"packages": 16
},
{
"context": "('block',)",
"packages": 16
},
{
"context": "('runTest', 'launch')",
"packages": 16
},
{
"context": "('require', 'require')",
"packages": 15
},
{
"context": "('check', 'check')",
"packages": 15
},
{
"context": "('repeat', 'repeat')",
"packages": 14
},
{
"context": "('runBlocking', 'expect')",
"packages": 13
}
]
},
"k3": {
"contexts": 3434,
"sore_successes": 546,
"total_methods": 6720,
"methods_in_good": 2062,
"coverage": 30.7,
"cross_package_contexts": 524,
"elapsed": 0.013,
"top_groups": [
{
"context": "('_eps',)",
"label": "raw_k3",
"methods": 161,
"unique": 1,
"unique_ratio": 0.006,
"alphabet_size": 0,
"sore": null,
"sore_success": false
},
{
"context": "('expectUnreached',)",
"label": "raw_k3",
"methods": 76,
"unique": 1,
"unique_ratio": 0.013,
"alphabet_size": 1,
"sore": "expectUnreached",
"sore_success": true
},
{
"context": "('runTest', 'expect', 'launch')",
"label": "raw_k3",
"methods": 72,
"unique": 67,
"unique_ratio": 0.931,
"alphabet_size": 64,
"sore": null,
"sore_success": false
},
{
"context": "('runTest', 'flow', 'emit')",
"label": "raw_k3",
"methods": 58,
"unique": 58,
"unique_ratio": 1.0,
"alphabet_size": 65,
"sore": null,
"sore_success": false
},
{
"context": "('test', 'main', 'verifyLines')",
"label": "raw_k3",
"methods": 56,
"unique": 1,
"unique_ratio": 0.018,
"alphabet_size": 3,
"sore": "test.main.verifyLines",
"sore_success": true
},
{
"context": "('runTest', 'flow', 'expect')",
"label": "raw_k3",
"methods": 52,
"unique": 49,
"unique_ratio": 0.942,
"alphabet_size": 68,
"sore": null,
"sore_success": false
},
{
"context": "('runTest', 'listOf', 'listOf')",
"label": "raw_k3",
"methods": 44,
"unique": 41,
"unique_ratio": 0.932,
"alphabet_size": 73,
"sore": null,
"sore_success": false
},
{
"context": "('expect',)",
"label": "raw_k3",
"methods": 42,
"unique": 1,
"unique_ratio": 0.024,
"alphabet_size": 1,
"sore": "expect",
"sore_success": true
},
{
"context": "('close',)",
"label": "raw_k3",
"methods": 40,
"unique": 1,
"unique_ratio": 0.025,
"alphabet_size": 1,
"sore": "close",
"sore_success": true
},
{
"context": "('runTest', 'currentDispatcher', 'asScheduler')",
"label": "raw_k3",
"methods": 40,
"unique": 16,
"unique_ratio": 0.4,
"alphabet_size": 17,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('_eps',)",
"packages": 48
},
{
"context": "('close',)",
"packages": 16
},
{
"context": "('block',)",
"packages": 16
},
{
"context": "('error', 'error')",
"packages": 13
},
{
"context": "('UnsupportedOperationException',)",
"packages": 13
},
{
"context": "('cancel',)",
"packages": 11
},
{
"context": "('toString',)",
"packages": 11
},
{
"context": "('dispatch',)",
"packages": 11
},
{
"context": "('runTest', 'listOf', 'listOf')",
"packages": 11
},
{
"context": "('runTest', 'repeat', 'repeat')",
"packages": 9
}
]
}
},
"coarsened": {
"seq_count": 6720,
"alphabet_size": 3759,
"alphabet_sample": [
"",
"(\n val data: String\n )",
"(\"un",
"()",
"() }",
"() } }",
"())",
"()) }",
"(),",
"(), 1.1)",
"(), self",
"().availab",
"(0)",
"(0),",
"(0, 100)",
"(0, 1024",
"(1)",
"(2)",
"(2,",
"(3)",
"(Bal",
"(Ch",
"(Sta",
"(Sto",
"(Thread",
"(Uni",
"(actorSta",
"(block: Runnable)",
"(clauseObje",
"(cou"
],
"k1": {
"contexts": 985,
"sore_successes": 118,
"total_methods": 6720,
"methods_in_good": 472,
"coverage": 7.0,
"cross_package_contexts": 320,
"elapsed": 0.008,
"top_groups": [
{
"context": "('runTest',)",
"label": "coarsened_k1",
"methods": 1691,
"unique": 1561,
"unique_ratio": 0.923,
"alphabet_size": 923,
"sore": null,
"sore_success": false
},
{
"context": "('runBlocking',)",
"label": "coarsened_k1",
"methods": 363,
"unique": 290,
"unique_ratio": 0.799,
"alphabet_size": 434,
"sore": null,
"sore_success": false
},
{
"context": "('IF',)",
"label": "coarsened_k1",
"methods": 351,
"unique": 309,
"unique_ratio": 0.88,
"alphabet_size": 727,
"sore": null,
"sore_success": false
},
{
"context": "('RETURN',)",
"label": "coarsened_k1",
"methods": 182,
"unique": 114,
"unique_ratio": 0.626,
"alphabet_size": 282,
"sore": null,
"sore_success": false
},
{
"context": "('expect',)",
"label": "coarsened_k1",
"methods": 117,
"unique": 45,
"unique_ratio": 0.385,
"alphabet_size": 95,
"sore": null,
"sore_success": false
},
{
"context": "('LOOP',)",
"label": "coarsened_k1",
"methods": 91,
"unique": 82,
"unique_ratio": 0.901,
"alphabet_size": 227,
"sore": null,
"sore_success": false
},
{
"context": "('EXCEPTION',)",
"label": "coarsened_k1",
"methods": 90,
"unique": 72,
"unique_ratio": 0.8,
"alphabet_size": 197,
"sore": null,
"sore_success": false
},
{
"context": "('test',)",
"label": "coarsened_k1",
"methods": 88,
"unique": 88,
"unique_ratio": 1.0,
"alphabet_size": 108,
"sore": null,
"sore_success": false
},
{
"context": "('expectUnreached',)",
"label": "coarsened_k1",
"methods": 76,
"unique": 3,
"unique_ratio": 0.039,
"alphabet_size": 2,
"sore": null,
"sore_success": false
},
{
"context": "('flow',)",
"label": "coarsened_k1",
"methods": 57,
"unique": 34,
"unique_ratio": 0.596,
"alphabet_size": 64,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('IF',)",
"packages": 71
},
{
"context": "('RETURN',)",
"packages": 54
},
{
"context": "('runTest',)",
"packages": 45
},
{
"context": "('EXCEPTION',)",
"packages": 41
},
{
"context": "('runBlocking',)",
"packages": 39
},
{
"context": "('LOOP',)",
"packages": 32
},
{
"context": "('value',)",
"packages": 21
},
{
"context": "('close',)",
"packages": 20
},
{
"context": "('suspendCancellableCoroutine',)",
"packages": 19
},
{
"context": "('block',)",
"packages": 18
}
]
},
"k2": {
"contexts": 2270,
"sore_successes": 256,
"total_methods": 6720,
"methods_in_good": 952,
"coverage": 14.2,
"cross_package_contexts": 493,
"elapsed": 0.008,
"top_groups": [
{
"context": "('runTest', 'expect')",
"label": "coarsened_k2",
"methods": 356,
"unique": 325,
"unique_ratio": 0.913,
"alphabet_size": 348,
"sore": null,
"sore_success": false
},
{
"context": "('runTest', 'flow')",
"label": "coarsened_k2",
"methods": 138,
"unique": 135,
"unique_ratio": 0.978,
"alphabet_size": 127,
"sore": null,
"sore_success": false
},
{
"context": "('runTest', 'Channel')",
"label": "coarsened_k2",
"methods": 112,
"unique": 108,
"unique_ratio": 0.964,
"alphabet_size": 152,
"sore": null,
"sore_success": false
},
{
"context": "('test', 'coroutines')",
"label": "coarsened_k2",
"methods": 88,
"unique": 88,
"unique_ratio": 1.0,
"alphabet_size": 108,
"sore": null,
"sore_success": false
},
{
"context": "('runBlocking', 'Unit')",
"label": "coarsened_k2",
"methods": 80,
"unique": 63,
"unique_ratio": 0.787,
"alphabet_size": 141,
"sore": null,
"sore_success": false
},
{
"context": "('runTest', 'flowOf')",
"label": "coarsened_k2",
"methods": 80,
"unique": 77,
"unique_ratio": 0.963,
"alphabet_size": 125,
"sore": null,
"sore_success": false
},
{
"context": "('IF', 'RETURN')",
"label": "coarsened_k2",
"methods": 75,
"unique": 70,
"unique_ratio": 0.933,
"alphabet_size": 211,
"sore": null,
"sore_success": false
},
{
"context": "('runBlocking', 'expect')",
"label": "coarsened_k2",
"methods": 75,
"unique": 55,
"unique_ratio": 0.733,
"alphabet_size": 98,
"sore": null,
"sore_success": false
},
{
"context": "('expectUnreached',)",
"label": "coarsened_k2",
"methods": 67,
"unique": 1,
"unique_ratio": 0.015,
"alphabet_size": 1,
"sore": "expectUnreached",
"sore_success": true
},
{
"context": "('RETURN', 'EXCEPTION')",
"label": "coarsened_k2",
"methods": 51,
"unique": 15,
"unique_ratio": 0.294,
"alphabet_size": 39,
"sore": null,
"sore_success": false
}
],
"cross_pkg_examples": [
{
"context": "('IF', 'RETURN')",
"packages": 27
},
{
"context": "('runTest', 'expect')",
"packages": 25
},
{
"context": "('RETURN', 'EXCEPTION')",
"packages": 24
},
{
"context": "('close', 'close')",
"packages": 19
},
{
"context": "('runTest', 'launch')",
"packages": 16
},
{
"context": "('require', 'require')",
"packages": 15
},
{
"context": "('IF', 'IF')",
"packages": 15
},
{
"context": "('runBlocking', 'expect')",
"packages": 13
},
{
"context": "('error', 'error')",
"packages": 13
},
{
"context": "('cancel', 'cancel')",
"packages": 13
}
]
},
"k3": {
"contexts": 3414,
"sore_successes": 326,
"total_methods": 6720,
"methods_in_good": 1282,
"coverage": 19.1,
"cross_package_contexts": 539,
"elapsed": 0.004,
"top_groups": [
{
"context": "('test', 'coroutines', 'guide')",
"label": "coarsened_k3",
"methods": 81,
"unique": 81,
"unique_ratio": 1.0,
"alphabet_size": 100,
"sore": null,
"sore_success": false
},
{
"context": "('runTest', 'Channel', 'Int')",
"label": "coarsened_k3",
"methods": 68,
"unique": 64,
"unique_ratio": 0.941,
"alphabet_size": 87,
"sore": null,
"sore_success": false
},
{
"context": "('expectUnreached',)",
"label": "coarsened_k3",
"methods": 67,
"unique": 1,
"unique_ratio": 0.015,
"alphabet_size": 1,
"sore": "expectUnreached",
"sore_success": true
},
{
"context": "('runTest', 'expect', 'launch')",
"label": "coarsened_k3",
"methods": 50,
"unique": 50,
"unique_ratio": 1.0,
"alphabet_size": 61,
"sore": null,
"sore_success": false
},
{
"context": "('runTest', 'flow', 'emit')",
"label": "coarsened_k3",
"methods": 46,
"unique": 46,
"unique_ratio": 1.0,
"alphabet_size": 66,
"sore": null,
"sore_success": false
},
{
"context": "('runTest', 'currentDispatcher', 'CoroutineDispatcher')",
"label": "coarsened_k3",
"methods": 40,
"unique": 16,
"unique_ratio": 0.4,
"alphabet_size": 22,
"sore": null,
"sore_success": false
},
{
"context": "('expect',)",
"label": "coarsened_k3",
"methods": 39,
"unique": 1,
"unique_ratio": 0.026,
"alphabet_size": 1,
"sore": "expect",
"sore_success": true
},
{
"context": "('runTest', 'flow', 'expect')",
"label": "coarsened_k3",
"methods": 38,
"unique": 36,
"unique_ratio": 0.947,
"alphabet_size": 78,
"sore": null,
"sore_success": false
},
{
"context": "('runTest', 'listOf', 'listOf')",
"label": "coarsened_k3",
"methods": 37,
"unique": 34,
"unique_ratio": 0.919,
"alphabet_size": 90,
"sore": null,
"sore_success": false
},
{
"context": "('noImpl',)",
"label": "coarsened_k3",
"methods": 34,
"unique": 1,
"unique_ratio": 0.029,
"alphabet_size": 1,
"sore": "noImpl",
"sore_success": true
}
],
"cross_pkg_examples": [
{
"context": "('close', 'close')",
"packages": 14
},
{
"context": "('RETURN', 'EXCEPTION', 'UnsupportedOperationException')",
"packages": 13
},
{
"context": "('error', 'error')",
"packages": 12
},
{
"context": "('value',)",
"packages": 12
},
{
"context": "('dispatch', 'dispatch')",
"packages": 11
},
{
"context": "('runTest', 'listOf', 'listOf')",
"packages": 11
},
{
"context": "('RETURN',)",
"packages": 10
},
{
"context": "('IF', 'RETURN', 'EXCEPTION')",
"packages": 10
},
{
"context": "('runTest', 'repeat', 'repeat')",
"packages": 9
},
{
"context": "('IF', 'IF')",
"packages": 9
}
]
}
}
}
Reference in a new issue Copy permalink