ommlds 0.0.0.dev484__tar.gz → 0.0.0.dev486__tar.gz

{ommlds-0.0.0.dev484/ommlds.egg-info → ommlds-0.0.0.dev486}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ommlds
-Version: 0.0.0.dev484
+Version: 0.0.0.dev486
 Summary: ommlds
 Author: wrmsr
 License-Expression: BSD-3-Clause
@@ -14,9 +14,9 @@ Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: omdev==0.0.0.dev484
-Requires-Dist: omlish==0.0.0.dev484
+Requires-Dist: omlish==0.0.0.dev486
 Provides-Extra: all
+Requires-Dist: omdev==0.0.0.dev486; extra == "all"
 Requires-Dist: llama-cpp-python~=0.3; extra == "all"
 Requires-Dist: mlx~=0.30; sys_platform == "darwin" and extra == "all"
 Requires-Dist: mlx-lm~=0.28; sys_platform == "darwin" and extra == "all"
@@ -37,6 +37,8 @@ Requires-Dist: ddgs~=9.9; extra == "all"
 Requires-Dist: mwparserfromhell~=0.7; extra == "all"
 Requires-Dist: wikitextparser~=0.56; extra == "all"
 Requires-Dist: lxml>=5.3; python_version < "3.13" and extra == "all"
+Provides-Extra: omdev
+Requires-Dist: omdev==0.0.0.dev486; extra == "omdev"
 Provides-Extra: backends
 Requires-Dist: llama-cpp-python~=0.3; extra == "backends"
 Requires-Dist: mlx~=0.30; sys_platform == "darwin" and extra == "backends"

{ommlds-0.0.0.dev484 → ommlds-0.0.0.dev486}/ommlds/.omlish-manifests.json

@@ -873,7 +873,7 @@
     "module": ".tools.git",
     "attr": null,
     "file": "ommlds/tools/git.py",
-    "line": 189,
+    "line": 190,
     "value": {
       "!omdev.tools.git.messages.GitMessageGeneratorManifest": {
         "module": "ommlds.tools.git",

{ommlds-0.0.0.dev484 → ommlds-0.0.0.dev486}/ommlds/__about__.py

@@ -8,13 +8,16 @@ class Project(ProjectBase):
     description = 'ommlds'
 
     dependencies = [
-        f'omdev == {__version__}',
         f'omlish == {__version__}',
     ]
 
     optional_dependencies = {
+        'omdev': [
+            f'omdev == {__version__}',
+        ],
+
         'backends': [
-            # 'diffusers ~= 0.35',
+            # 'diffusers ~= 0.36',
 
             'llama-cpp-python ~= 0.3',
 

{ommlds-0.0.0.dev484 → ommlds-0.0.0.dev486}/ommlds/nanochat/tokenizers.py

@@ -18,7 +18,10 @@ from omlish import lang
 
 with lang.auto_proxy_import(globals()):
     import tiktoken
-    import tokenizers
+    import tokenizers.decoders
+    import tokenizers.models
+    import tokenizers.pre_tokenizers
+    import tokenizers.trainers
 
 
 rustbpe: ta.Any = lang.proxy_import('.rustbpe', __package__)
@@ -27,7 +30,7 @@ rustbpe: ta.Any = lang.proxy_import('.rustbpe', __package__)
 ##
 
 
-SPECIAL_TOKENS = [
+SPECIAL_TOKENS: ta.Sequence[str] = [
     # every document begins with the Beginning of Sequence (BOS) token that delimits documents
     '<|bos|>',
     # tokens below are only used during finetuning to render Conversations into token ids
@@ -45,10 +48,18 @@ SPECIAL_TOKENS = [
 # NOTE: this split pattern deviates from GPT-4 in that we use \p{N}{1,2} instead of \p{N}{1,3}
 # I did this because I didn't want to "waste" too many tokens on numbers for smaller vocab sizes.
 # I haven't validated that this is actually a good idea, TODO.
-SPLIT_PATTERN = r"""'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}+|\p{N}{1,2}| ?[^\s\p{L}\p{N}]++[\r\n]*|\s*[\r\n]|\s+(?!\S)|\s+"""  # noqa
+SPLIT_PATTERN = (
+    r"'(?i:[sdmt]|ll|ve|re)|"
+    r"[^\r\n\p{L}\p{N}]?+\p{L}+|"
+    r"\p{N}{1,2}|"
+    r" ?[^\s\p{L}\p{N}]++[\r\n]*|"
+    r"\s*[\r\n]|"
+    r"\s+(?!\S)|"
+    r"\s+"
+)
 
 
-# -----------------------------------------------------------------------------
+##
 # Generic GPT-4-style tokenizer based on HuggingFace Tokenizer
 
 
@@ -87,22 +98,28 @@ class HuggingFaceTokenizer:
             unk_token=None,
             fuse_unk=False,
         ))
+
         # Normalizer: None
         tokenizer.normalizer = None
+
         # Pre-tokenizer: GPT-4 style
         # the regex pattern used by GPT-4 to split text into groups before BPE
         # NOTE: The pattern was changed from \p{N}{1,3} to \p{N}{1,2} because I suspect it is harmful to
         # very small models and smaller vocab sizes, because it is a little bit wasteful in the token space.
         # (but I haven't validated this! TODO)
         gpt4_split_regex = tokenizers.Regex(split_pattern)  # huggingface demands that you wrap it in Regex!!
+
         tokenizer.pre_tokenizer = tokenizers.pre_tokenizers.Sequence([
             tokenizers.pre_tokenizers.Split(pattern=gpt4_split_regex, behavior='isolated', invert=False),
             tokenizers.pre_tokenizers.ByteLevel(add_prefix_space=False, use_regex=False),
         ])
+
         # Decoder: ByteLevel (it pairs together with the ByteLevel pre-tokenizer)
         tokenizer.decoder = tokenizers.decoders.ByteLevel()
+
         # Post-processor: None
         tokenizer.post_processor = None
+
         # Trainer: BPE
         trainer = tokenizers.trainers.BpeTrainer(
             vocab_size=vocab_size,
@@ -111,8 +128,10 @@ class HuggingFaceTokenizer:
             initial_alphabet=tokenizers.pre_tokenizers.ByteLevel.alphabet(),
             special_tokens=special_tokens,
         )
+
         # Kick off the training
         tokenizer.train_from_iterator(text_iterator, trainer)
+
         return cls(tokenizer)
 
     def encode_ordinary(self, text):
@@ -174,7 +193,7 @@ class HuggingFaceTokenizer:
         print(f'Saved tokenizer to {tokenizer_path}')
 
 
-# -----------------------------------------------------------------------------
+##
 # Tokenizer based on rustbpe + tiktoken combo
 
 
@@ -255,6 +274,7 @@ class RustBPETokenizer:
                 ids.insert(0, prepend_id)  # TODO: slightly inefficient here? :( hmm
             if append is not None:
                 ids.append(append_id)
+
         elif isinstance(text, list):
             ids = self.enc.encode_ordinary_batch(text, num_threads=num_threads)
             if prepend is not None:
@@ -263,6 +283,7 @@ class RustBPETokenizer:
             if append is not None:
                 for ids_row in ids:
                     ids_row.append(append_id)
+
         else:
             raise ValueError(f'Invalid input type: {type(text)}')  # noqa
 
@@ -285,6 +306,7 @@ class RustBPETokenizer:
     def render_conversation(self, conversation, max_tokens=2048):
         """
         Tokenize a single Chat conversation (which we call a "doc" or "document" here).
+
         Returns:
         - ids: list[int] is a list of token ids of this rendered conversation
         - mask: list[int] of same length, mask = 1 for tokens that the Assistant is expected to train on.
@@ -324,7 +346,10 @@ class RustBPETokenizer:
         for i, message in enumerate(messages):
             # some sanity checking here around assumptions, to prevent footguns
             must_be_from = 'user' if i % 2 == 0 else 'assistant'
-            check.state(message['role'] == must_be_from, f"Message {i} is from {message['role']} but should be from {must_be_from}")  # noqa
+            check.state(
+                message['role'] == must_be_from,
+                f"Message {i} is from {message['role']} but should be from {must_be_from}",
+            )
 
             # content can be either a simple string or a list of parts (e.g. containing tool calls)
             content = message['content']
@@ -335,33 +360,42 @@ class RustBPETokenizer:
                 add_tokens(user_start, 0)
                 add_tokens(value_ids, 0)
                 add_tokens(user_end, 0)
+
             elif message['role'] == 'assistant':
                 add_tokens(assistant_start, 0)
+
                 if isinstance(content, str):
                     # simple string => simply add the tokens
                     value_ids = self.encode(content)
                     add_tokens(value_ids, 1)
+
                 elif isinstance(content, list):
                     for part in content:
                         value_ids = self.encode(part['text'])
+
                         if part['type'] == 'text':
                             # string part => simply add the tokens
                             add_tokens(value_ids, 1)
+
                         elif part['type'] == 'python':
                             # python tool call => add the tokens inside <|python_start|> and <|python_end|>
                             add_tokens(python_start, 1)
                             add_tokens(value_ids, 1)
                             add_tokens(python_end, 1)
+
                         elif part['type'] == 'python_output':
                             # python output => add the tokens inside <|output_start|> and <|output_end|>
                             # none of these tokens are supervised because the tokens come from Python at test time
                             add_tokens(output_start, 0)
                             add_tokens(value_ids, 0)
                             add_tokens(output_end, 0)
+
                         else:
                             raise ValueError(f"Unknown part type: {part['type']}")
+
                 else:
                     raise ValueError(f'Unknown content type: {type(content)}')
+
                 add_tokens(assistant_end, 1)
 
         # truncate to max_tokens tokens MAX (helps prevent OOMs)

{ommlds-0.0.0.dev484 → ommlds-0.0.0.dev486}/ommlds/tools/git.py

@@ -12,6 +12,7 @@ import urllib.request
 
 from omdev.git.magic import GIT_DIFF_OMIT_MAGIC_COMMENT
 from omdev.home.secrets import load_secrets
+from omdev.py.srcheaders import get_py_header_lines
 from omdev.tools.git.messages import GitMessageGenerator
 from omlish import check
 from omlish import lang
@@ -216,9 +217,14 @@ class AiGitMessageGenerator(GitMessageGenerator):
         except FileNotFoundError:
             return False
 
+        hls = get_py_header_lines(f_src)
+
         return any(
-            l.startswith(GIT_DIFF_OMIT_MAGIC_COMMENT)
-            for l in f_src.splitlines()
+            hl.src.strip() in (
+                '# @omlish-generated',
+                GIT_DIFF_OMIT_MAGIC_COMMENT,
+            )
+            for hl in hls
         )
 
     # TODO: configurable

{ommlds-0.0.0.dev484 → ommlds-0.0.0.dev486/ommlds.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ommlds
-Version: 0.0.0.dev484
+Version: 0.0.0.dev486
 Summary: ommlds
 Author: wrmsr
 License-Expression: BSD-3-Clause
@@ -14,9 +14,9 @@ Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: omdev==0.0.0.dev484
-Requires-Dist: omlish==0.0.0.dev484
+Requires-Dist: omlish==0.0.0.dev486
 Provides-Extra: all
+Requires-Dist: omdev==0.0.0.dev486; extra == "all"
 Requires-Dist: llama-cpp-python~=0.3; extra == "all"
 Requires-Dist: mlx~=0.30; sys_platform == "darwin" and extra == "all"
 Requires-Dist: mlx-lm~=0.28; sys_platform == "darwin" and extra == "all"
@@ -37,6 +37,8 @@ Requires-Dist: ddgs~=9.9; extra == "all"
 Requires-Dist: mwparserfromhell~=0.7; extra == "all"
 Requires-Dist: wikitextparser~=0.56; extra == "all"
 Requires-Dist: lxml>=5.3; python_version < "3.13" and extra == "all"
+Provides-Extra: omdev
+Requires-Dist: omdev==0.0.0.dev486; extra == "omdev"
 Provides-Extra: backends
 Requires-Dist: llama-cpp-python~=0.3; extra == "backends"
 Requires-Dist: mlx~=0.30; sys_platform == "darwin" and extra == "backends"

{ommlds-0.0.0.dev484 → ommlds-0.0.0.dev486}/ommlds.egg-info/requires.txt

@@ -1,7 +1,7 @@
-omdev==0.0.0.dev484
-omlish==0.0.0.dev484
+omlish==0.0.0.dev486
 
 [all]
+omdev==0.0.0.dev486
 llama-cpp-python~=0.3
 tiktoken~=0.12
 tinygrad~=0.11
@@ -54,6 +54,9 @@ numpy>=1.26
 pytesseract~=0.3
 rapidocr-onnxruntime~=1.4
 
+[omdev]
+omdev==0.0.0.dev486
+
 [pillow]
 pillow~=12.0
 

{ommlds-0.0.0.dev484 → ommlds-0.0.0.dev486}/pyproject.toml

@@ -13,7 +13,7 @@ urls = {source = 'https://github.com/wrmsr/omlish'}
 license = 'BSD-3-Clause'
 readme = 'README.md'
 requires-python = '>=3.13'
-version = '0.0.0.dev484'
+version = '0.0.0.dev486'
 classifiers = [
     'Development Status :: 2 - Pre-Alpha',
     'Intended Audience :: Developers',
@@ -24,12 +24,12 @@ classifiers = [
 ]
 description = 'ommlds'
 dependencies = [
-    'omdev == 0.0.0.dev484',
-    'omlish == 0.0.0.dev484',
+    'omlish == 0.0.0.dev486',
 ]
 
 [project.optional-dependencies]
 all = [
+    'omdev == 0.0.0.dev486',
     'llama-cpp-python ~= 0.3',
     'mlx ~= 0.30; sys_platform == "darwin"',
     'mlx-lm ~= 0.28; sys_platform == "darwin"',
@@ -51,6 +51,9 @@ all = [
     'wikitextparser ~= 0.56',
     'lxml >= 5.3; python_version < "3.13"',
 ]
+omdev = [
+    'omdev == 0.0.0.dev486',
+]
 backends = [
     'llama-cpp-python ~= 0.3',
     'mlx ~= 0.30; sys_platform == "darwin"',