9d206f586f
- Plan: match rate audit, decomposition A/B, distributional A/B, fragment quality - Golden config: best-known values for all heuristic parameters - Single source of truth in bex/golden_config.py
64 lines
2.3 KiB
Python
64 lines
2.3 KiB
Python
"""Golden config — best-known heuristic values for the grammar inference pipeline.
|
|
|
|
This file is the single source of truth for optimal parameter values.
|
|
Import it when running evaluations or setting up new codebases.
|
|
|
|
Last updated: Round 19 (Crucio evaluation)
|
|
"""
|
|
|
|
# Core pipeline
|
|
MIN_COVERAGE = 0.05 # BEX outlier threshold (was 0.8, too aggressive)
|
|
MIN_METHODS = 3 # Min methods per group (was 5, lost too many)
|
|
SCORING_METHOD = "langsize" # Language Size (Bex et al. arXiv:1004.2372)
|
|
|
|
# Grouping
|
|
SLICE = "package" # Per-directory (not flat, not reduce)
|
|
SPLIT_MIXED = True # Recursive split by first symbol
|
|
MAX_DEPTH = 3 # Max recursion depth for split
|
|
CLUSTER_METHOD = "first-symbol" # Split method (distributional = no improvement)
|
|
|
|
# Quality filter
|
|
MIN_STRUCTURE = 0.5 # Drop flat bags (noise)
|
|
MAX_MDL = 200.0 # Drop high-score grammars
|
|
|
|
# Decomposition (Crucio Phase 2)
|
|
DECOMPOSE = True # Break long sequences into fragments
|
|
MAX_SEQ_LENGTH = 4 # Max fragment length (5 = too aggressive, 4 = sweet spot)
|
|
|
|
# Algorithms
|
|
CRX_METHOD = "standard" # Standard CRX (refined = trivial on large groups)
|
|
INCLUDE_KORE = False # kORE = slow, no improvement
|
|
INCLUDE_IDREGEX = False # iDRegEx = slow, rare benefit
|
|
IDREGEX_REFINE = False # iDRegEx refinement = rare benefit
|
|
|
|
|
|
def get_golden_config():
|
|
"""Return golden config as a dict for easy passing to analyze_directory()."""
|
|
return {
|
|
"min_coverage": MIN_COVERAGE,
|
|
"min_methods": MIN_METHODS,
|
|
"method": SCORING_METHOD,
|
|
"slice": SLICE,
|
|
"split_mixed": SPLIT_MIXED,
|
|
"cluster_method": CLUSTER_METHOD,
|
|
"min_structure": MIN_STRUCTURE,
|
|
"decompose": DECOMPOSE,
|
|
"max_seq_length": MAX_SEQ_LENGTH,
|
|
"crx_method": CRX_METHOD,
|
|
"include_kore": INCLUDE_KORE,
|
|
"include_idregex": INCLUDE_IDREGEX,
|
|
"idregex_refine": IDREGEX_REFINE,
|
|
}
|
|
|
|
|
|
def apply_to_directory(dir_path, include=None, main_only=True, **overrides):
|
|
"""Run analyze_directory with golden config. Override any parameter."""
|
|
from bex.tag_preprocessor.analyze import analyze_directory
|
|
config = get_golden_config()
|
|
config.update(overrides)
|
|
return analyze_directory(
|
|
dir_path,
|
|
include=include,
|
|
main_only=main_only,
|
|
**config,
|
|
)
|