diff --git a/bex/grammar.py b/bex/grammar.py index ab4db66..94e119e 100644 --- a/bex/grammar.py +++ b/bex/grammar.py @@ -170,7 +170,7 @@ def _match_rep(child, seq, pos, min_rep): # Counting (for MDL scoring) # --------------------------------------------------------------------------- -_COUNT_CAP = 10**12 +_COUNT_CAP = 10 ** 30 @lru_cache(maxsize=None) def count_words(node, length): diff --git a/bex/tag_preprocessor/analyze.py b/bex/tag_preprocessor/analyze.py index e8e01d5..29e8dc5 100644 --- a/bex/tag_preprocessor/analyze.py +++ b/bex/tag_preprocessor/analyze.py @@ -757,8 +757,8 @@ def analyze_directory( crx_method='standard', min_structure=0.0, context_strategy="dir", - decompose=False, - max_seq_length=5, + decompose=True, + max_seq_length=4, reduce_threshold=0.15, split_mixed=False, idregex_refine=False, @@ -1040,12 +1040,16 @@ def _parse_args(argv=None): help="Method to split mixed groups: first-symbol (fast, crude) or distributional (slower, smarter clustering)", ) parser.add_argument( - "--decompose", action="store_true", - help="Decompose long sequences into shorter fragments before inference", + "--decompose", action="store_true", default=True, + help="Decompose long sequences into shorter fragments before inference (default: on)", ) parser.add_argument( - "--max-seq-length", type=int, default=5, - help="Maximum sequence length after decomposition (default: 5)", + "--no-decompose", dest="decompose", action="store_false", + help="Disable sequence decomposition", + ) + parser.add_argument( + "--max-seq-length", type=int, default=4, + help="Maximum sequence length after decomposition (default: 4)", ) parser.add_argument( "--idregex-refine", action="store_true",