feat: default decomposition ON + restore Language Size scoring discrimination
B (decompose default): - analyze_directory + CLI now default decompose=True (--no-decompose to disable); default max_seq_length 5 -> 4 (matches golden_config). - Decomposing long sequences into <=4-symbol fragments yields tighter grammars and disables the too_diverse skip, so far more packages get a grammar. RAGSAK: 29 -> 95 grammars, pure bags 9 -> 5; fastapi 74 -> 26 pure bags; zod 15 -> 5. All runs exit 0, no stalls. A (repair Language Size scorer, not a new one): - _COUNT_CAP was applied to count_words' RETURN value, silently clamping lang_size_score / model_cost / data_cost at 10^12 for every real codebase grammar. That broke ADR-13 (Language Size picks most-specific grammar): bags and tight grammars tied at 10^12, so the scorer could not prefer specific grammars over generic ones. - _COUNT_CAP raised to 10**30. Memoization (_count_concat, earlier commit) already prevents the recursion hang the cap was guarding against, so the cap no longer needs to clamp scores. lang_size_score now discriminates (verified: tight=20 vs bag=9975).
This commit is contained in:
parent
1810a695d0
commit
8f412905c5
2 changed files with 11 additions and 7 deletions
|
|
@ -170,7 +170,7 @@ def _match_rep(child, seq, pos, min_rep):
|
||||||
# Counting (for MDL scoring)
|
# Counting (for MDL scoring)
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
_COUNT_CAP = 10**12
|
_COUNT_CAP = 10 ** 30
|
||||||
|
|
||||||
@lru_cache(maxsize=None)
|
@lru_cache(maxsize=None)
|
||||||
def count_words(node, length):
|
def count_words(node, length):
|
||||||
|
|
|
||||||
|
|
@ -757,8 +757,8 @@ def analyze_directory(
|
||||||
crx_method='standard',
|
crx_method='standard',
|
||||||
min_structure=0.0,
|
min_structure=0.0,
|
||||||
context_strategy="dir",
|
context_strategy="dir",
|
||||||
decompose=False,
|
decompose=True,
|
||||||
max_seq_length=5,
|
max_seq_length=4,
|
||||||
reduce_threshold=0.15,
|
reduce_threshold=0.15,
|
||||||
split_mixed=False,
|
split_mixed=False,
|
||||||
idregex_refine=False,
|
idregex_refine=False,
|
||||||
|
|
@ -1040,12 +1040,16 @@ def _parse_args(argv=None):
|
||||||
help="Method to split mixed groups: first-symbol (fast, crude) or distributional (slower, smarter clustering)",
|
help="Method to split mixed groups: first-symbol (fast, crude) or distributional (slower, smarter clustering)",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--decompose", action="store_true",
|
"--decompose", action="store_true", default=True,
|
||||||
help="Decompose long sequences into shorter fragments before inference",
|
help="Decompose long sequences into shorter fragments before inference (default: on)",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--max-seq-length", type=int, default=5,
|
"--no-decompose", dest="decompose", action="store_false",
|
||||||
help="Maximum sequence length after decomposition (default: 5)",
|
help="Disable sequence decomposition",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--max-seq-length", type=int, default=4,
|
||||||
|
help="Maximum sequence length after decomposition (default: 4)",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--idregex-refine", action="store_true",
|
"--idregex-refine", action="store_true",
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue