mostlyai-engine 1.4.3__tar.gz → 1.4.5__tar.gz

{mostlyai_engine-1.4.3 → mostlyai_engine-1.4.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mostlyai-engine
-Version: 1.4.3
+Version: 1.4.5
 Summary: Synthetic Data Engine
 Project-URL: homepage, https://github.com/mostly-ai/mostlyai-engine
 Project-URL: repository, https://github.com/mostly-ai/mostlyai-engine
@@ -28,7 +28,7 @@ Requires-Dist: accelerate>=1.5.0
 Requires-Dist: datasets>=3.0.0
 Requires-Dist: huggingface-hub[hf-xet]>=0.30.2
 Requires-Dist: joblib>=1.4.2
-Requires-Dist: json-repair>=0.30.0
+Requires-Dist: json-repair<0.47.0,>=0.30.0
 Requires-Dist: numpy>=1.26.3
 Requires-Dist: opacus>=1.5.2
 Requires-Dist: pandas~=2.2.0

{mostlyai_engine-1.4.3 → mostlyai_engine-1.4.5}/mostlyai/engine/__init__.py

@@ -22,7 +22,7 @@ from mostlyai.engine.splitting import split
 from mostlyai.engine.training import train
 
 __all__ = ["split", "analyze", "encode", "train", "generate", "init_logging", "set_random_state"]
-__version__ = "1.4.3"
+__version__ = "1.4.5"
 
 # suppress specific warning related to os.fork() in multi-threaded processes
 warnings.filterwarnings("ignore", category=DeprecationWarning, message=".*multi-threaded.*fork.*")

{mostlyai_engine-1.4.3 → mostlyai_engine-1.4.5}/mostlyai/engine/_language/training.py

@@ -16,6 +16,7 @@ import gc
 import json
 import logging
 import math
+import os
 import time
 import warnings
 from collections.abc import Callable
@@ -431,6 +432,10 @@ def train(
                 model_config = LSTMFromScratchConfig(vocab_size=len(tokenizer), with_dp=with_dp)
                 model = LSTMFromScratchLMHeadModel(model_config).to(device)
         else:
+            # use MOSTLY_HUGGING_FACE_TOKEN if available
+            if os.getenv("MOSTLY_HUGGING_FACE_TOKEN"):
+                os.environ["HF_TOKEN"] = os.environ["MOSTLY_HUGGING_FACE_TOKEN"]
+
             model, model_config = load_base_model_and_config(
                 model_id_or_path,
                 device=device,

{mostlyai_engine-1.4.3 → mostlyai_engine-1.4.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mostlyai-engine"
-version = "1.4.3"
+version = "1.4.5"
 description = "Synthetic Data Engine"
 authors = [{ name = "MOSTLY AI", email = "dev@mostly.ai" }]
 requires-python = ">=3.10"
@@ -39,7 +39,7 @@ dependencies = [
     "huggingface-hub[hf-xet]>=0.30.2",
     "opacus>=1.5.2",  # switch to 1.5.4 (once released) to allow numpy 2
     "xgrammar>=0.1.18",  # for vllm 0.8.5.post1 compatibility
-    "json-repair>=0.30.0",
+    "json-repair>=0.30.0, <0.47.0",  # fixes errors in tests when building engine 1.4.4
     "torch>=2.6.0,<2.6.1",
     "torchaudio>=2.6.0,<2.6.1",  # for vllm 0.8.5.post1 compatibility
     "torchvision>=0.21.0,<0.21.1"  # for vllm 0.8.5.post1 compatibility
@@ -64,6 +64,7 @@ docs = [
     "mkdocs>=1.6",
     "mkdocstrings[crystal, python]>=0.29",
     "mkdocs-material>=9.0",
+    "mkdocs-llmstxt>=0.2",
     "griffe>=1.0",
     "pymdown-extensions>=10.0",
     "griffe-fieldz>=0.2",