tobi/grammar-inference-engine
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

feature/treesitter-tag-queries #2

Open
tobi wants to merge 78 commits from feature/treesitter-tag-queries into main
pull from: feature/treesitter-tag-queries
merge into: tobi:main
Conversation 0 Commits 78 Files changed 246 +11 -7
2 changed files with 11 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 8f412905c5 - Show all commits

2
bex/grammar.py
View file

@ -170,7 +170,7 @@ def _match_rep(child, seq, pos, min_rep):
# Counting (for MDL scoring) # Counting (for MDL scoring)
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
_COUNT_CAP = 10**12 _COUNT_CAP = 10 ** 30
@lru_cache(maxsize=None) @lru_cache(maxsize=None)
def count_words(node, length): def count_words(node, length):

16
bex/tag_preprocessor/analyze.py
View file

@ -757,8 +757,8 @@ def analyze_directory(
crx_method='standard', crx_method='standard',
min_structure=0.0, min_structure=0.0,
context_strategy="dir", context_strategy="dir",
decompose=False, decompose=True,
max_seq_length=5, max_seq_length=4,
reduce_threshold=0.15, reduce_threshold=0.15,
split_mixed=False, split_mixed=False,
idregex_refine=False, idregex_refine=False,
@ -1040,12 +1040,16 @@ def _parse_args(argv=None):
help="Method to split mixed groups: first-symbol (fast, crude) or distributional (slower, smarter clustering)", help="Method to split mixed groups: first-symbol (fast, crude) or distributional (slower, smarter clustering)",
) )
parser.add_argument( parser.add_argument(
"--decompose", action="store_true", "--decompose", action="store_true", default=True,
help="Decompose long sequences into shorter fragments before inference", help="Decompose long sequences into shorter fragments before inference (default: on)",
) )
parser.add_argument( parser.add_argument(
"--max-seq-length", type=int, default=5, "--no-decompose", dest="decompose", action="store_false",
help="Maximum sequence length after decomposition (default: 5)", help="Disable sequence decomposition",
)
parser.add_argument(
"--max-seq-length", type=int, default=4,
help="Maximum sequence length after decomposition (default: 4)",
) )
parser.add_argument( parser.add_argument(
"--idregex-refine", action="store_true", "--idregex-refine", action="store_true",