mostlyai-engine 2.4.0__tar.gz → 2.5.0__tar.gz

mostlyai_engine-2.5.0/LICENSE_HEADER

@@ -0,0 +1,13 @@
+Copyright 2025 MOSTLY AI
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

{mostlyai_engine-2.4.0 → mostlyai_engine-2.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mostlyai-engine
-Version: 2.4.0
+Version: 2.5.0
 Summary: Synthetic Data Engine
 Project-URL: homepage, https://github.com/mostly-ai/mostlyai-engine
 Project-URL: repository, https://github.com/mostly-ai/mostlyai-engine
@@ -8,6 +8,7 @@ Project-URL: documentation, https://mostly-ai.github.io/mostlyai-engine/
 Author-email: MOSTLY AI <dev@mostly.ai>
 License-Expression: Apache-2.0
 License-File: LICENSE
+License-File: LICENSE_HEADER
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Financial and Insurance Industry
@@ -17,13 +18,12 @@ Classifier: Intended Audience :: Science/Research
 Classifier: Intended Audience :: Telecommunications Industry
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
-Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Software Development :: Libraries
 Classifier: Typing :: Typed
-Requires-Python: >=3.10
+Requires-Python: <3.14,>=3.11
 Requires-Dist: accelerate>=1.5.0
 Requires-Dist: datasets>=3.0.0
 Requires-Dist: huggingface-hub[hf-xet]>=0.30.2
@@ -41,7 +41,7 @@ Requires-Dist: tokenizers>=0.21.0
 Requires-Dist: torch<2.10.0,>=2.9.0
 Requires-Dist: torchaudio<2.10.0,>=2.9.0
 Requires-Dist: torchvision<0.25.0,>=0.24.0
-Requires-Dist: transformers>=4.55.0
+Requires-Dist: transformers<5,>=4.55.0
 Requires-Dist: xgrammar>=0.1.21
 Provides-Extra: gpu
 Requires-Dist: bitsandbytes==0.42.0; (sys_platform == 'darwin') and extra == 'gpu'

{mostlyai_engine-2.4.0 → mostlyai_engine-2.5.0}/mostlyai/engine/__init__.py

@@ -34,7 +34,7 @@ __all__ = [
     "TabularARGN",
     "LanguageModel",
 ]
-__version__ = "2.4.0"
+__version__ = "2.5.0"
 
 # suppress specific warning related to os.fork() in multi-threaded processes
 warnings.filterwarnings("ignore", category=DeprecationWarning, message=".*multi-threaded.*fork.*")

{mostlyai_engine-2.4.0 → mostlyai_engine-2.5.0}/mostlyai/engine/_common.py

@@ -273,8 +273,9 @@ def safe_convert_datetime(values: pd.Series, date_only: bool = False) -> pd.Seri
         utc=True,
         dayfirst=False,  # assume 1/3/2020 is Jan 3
     )
-    # check whether firstday=True yields less non-NA, and if so, switch to using that flag
-    if values_parsed_fixed.isna().sum() > values.isna().sum():
+    has_slash_dates = values.astype("string").str.contains(r"^\s*\d{1,2}/\d{1,2}/\d{4}(?:\s|$)", regex=True).any()
+    # some mixed-format slash dates are interpreted more reliably with dayfirst=True
+    if has_slash_dates and values_parsed_fixed.isna().sum() > values.isna().sum():
         values_parsed_fixed_dayfirst = pd.to_datetime(
             values,
             errors="coerce",  # silently map invalid dates to NA
@@ -937,6 +938,7 @@ def impute_from_non_nan_distribution(values: pd.Series, column_stats: dict) -> t
     Returns:
         tuple[pd.Series, pd.Series]: The series with imputed values and the mask of NaNs.
     """
+    values = values.copy()
     nan_mask = values.isna()
     vc = values.value_counts(normalize=True)
     if vc.empty:
@@ -944,7 +946,8 @@ def impute_from_non_nan_distribution(values: pd.Series, column_stats: dict) -> t
     probs = vc.values
     categories = vc.index
     # NOTE: an alternative will be to use the largest remainder method
-    values[nan_mask] = np.random.choice(categories, size=nan_mask.sum(), p=probs)
+    if nan_mask.any():
+        values.loc[nan_mask] = np.random.choice(categories, size=nan_mask.sum(), p=probs)
     return values, nan_mask.astype(int)
 
 

{mostlyai_engine-2.4.0 → mostlyai_engine-2.5.0}/mostlyai/engine/_encoding_types/tabular/character.py

@@ -102,7 +102,9 @@ def encode_character(values: pd.Series, stats: dict, _: pd.Series | None = None)
     df_split = split_sub_columns_character(values, max_string_length)
     for idx in range(max_string_length):
         sub_col = f"P{idx}"
-        np_codes = np.array(pd.Categorical(df_split[sub_col], categories=stats["codes"][sub_col]).codes)
+        categories = list(stats["codes"][sub_col].keys())
+        values_at_pos = df_split[sub_col].where(df_split[sub_col].isin(categories), UNKNOWN_TOKEN)
+        np_codes = np.array(pd.Categorical(values_at_pos, categories=categories).codes)
         np.place(np_codes, np_codes == -1, 0)
         df_split[sub_col] = np_codes
     if stats["has_nan"]:

{mostlyai_engine-2.4.0 → mostlyai_engine-2.5.0}/mostlyai/engine/_encoding_types/tabular/datetime.py

@@ -234,9 +234,10 @@ def decode_datetime(df_encoded: pd.DataFrame, stats: dict):
     d = df_encoded["day"] + stats["min_values"]["day"]
     # fix invalid dates by setting these to last day of month
     is_leap = y.apply(lambda x: calendar.isleap(x))
-    d[is_leap & (m == 2) & (d > 29)] = 29
-    d[~is_leap & (m == 2) & (d > 28)] = 28
-    d[((m == 4) | (m == 6) | (m == 9) | (m == 11)) & (d > 30)] = 30
+    d = d.copy()
+    d.loc[is_leap & (m == 2) & (d > 29)] = 29
+    d.loc[~is_leap & (m == 2) & (d > 28)] = 28
+    d.loc[((m == 4) | (m == 6) | (m == 9) | (m == 11)) & (d > 30)] = 30
     # concatenate to datetime string
     y = y.astype(str)
     m = m.astype(str).str.zfill(2)
@@ -271,7 +272,7 @@ def decode_datetime(df_encoded: pd.DataFrame, stats: dict):
         # set all values to NaN if no valid values were present
         values[df_encoded["nan"] == 0] = pd.NA
     # convert from string to datetime
-    values = pd.to_datetime(values)
+    values = pd.to_datetime(values).astype("datetime64[ns]")
     if not stats["has_time"]:
-        values = pd.to_datetime(values.dt.date)
+        values = pd.to_datetime(values.dt.date).astype("datetime64[ns]")
     return values

{mostlyai_engine-2.4.0 → mostlyai_engine-2.5.0}/mostlyai/engine/_language/xgrammar_utils.py

@@ -75,14 +75,23 @@ def create_schemas(
     numeric_fields = field_types.get(ModelEncodingType.language_numeric, [])
     datetime_fields = field_types.get(ModelEncodingType.language_datetime, [])
     cache = {}
+
+    def _normalize_seed_value(seed_value):
+        return None if pd.isna(seed_value) else seed_value
+
     for _, seed_row in seed_df.iterrows():
-        cache_key = hash(tuple(sorted([(field_name, str(seed_value)) for field_name, seed_value in seed_row.items()])))
+        normalized_seed_items = [
+            (field_name, _normalize_seed_value(seed_value)) for field_name, seed_value in seed_row.items()
+        ]
+        cache_key = hash(
+            tuple(sorted([(field_name, str(seed_value)) for field_name, seed_value in normalized_seed_items]))
+        )
         if cache_key in cache:
             yield cache[cache_key]
             continue
         model_dict = {}
         if not seed_row.empty:
-            model_dict |= {field_name: (Literal[seed_value], ...) for field_name, seed_value in seed_row.items()}  # type: ignore[valid-type]
+            model_dict |= {field_name: (Literal[seed_value], ...) for field_name, seed_value in normalized_seed_items}  # type: ignore[valid-type]
         for field_name in unseeded_fields:
             if field_name in categorical_fields:
                 categories = stats["columns"][field_name]["categories"]

{mostlyai_engine-2.4.0 → mostlyai_engine-2.5.0}/mostlyai/engine/_tabular/common.py

@@ -197,7 +197,7 @@ def prepare_context_inputs(
     # Build flat context inputs (CTXFLT/*)
     ctxflt_inputs = {
         col: torch.unsqueeze(
-            torch.as_tensor(ctx_encoded[col].to_numpy(), device=device).type(torch.int),
+            torch.as_tensor(ctx_encoded[col].to_numpy(copy=True), device=device).type(torch.int),
             dim=-1,
         )
         for col in ctx_encoded.columns

{mostlyai_engine-2.4.0 → mostlyai_engine-2.5.0}/mostlyai/engine/_tabular/generation.py

@@ -640,10 +640,9 @@ def decode_buffered_samples(
             keys=keys,
             key_name=tgt_context_key,
         )
-        df_syn = df_syn.drop(
-            columns=[c for c in df_syn.columns if c.startswith(POSITIONAL_COLUMN)],
-            axis=1,
-        ).reset_index(drop=True)
+        df_syn = df_syn.drop(columns=[c for c in df_syn.columns if c.startswith(POSITIONAL_COLUMN)]).reset_index(
+            drop=True
+        )
     else:
         data, seed_data = zip(*buffer.buffer)
         df_syn = pd.concat(data, axis=0).reset_index(drop=True)
@@ -1224,7 +1223,9 @@ def generate(
                 # Use context inputs prepared earlier
                 x = ctx_inputs
                 fixed_values = {
-                    col: torch.as_tensor(seed_batch_encoded[col].to_numpy(), device=model.device).type(torch.int)
+                    col: torch.as_tensor(seed_batch_encoded[col].to_numpy(copy=True), device=model.device).type(
+                        torch.int
+                    )
                     for col in seed_batch_encoded.columns
                     if col in tgt_sub_columns
                 }

{mostlyai_engine-2.4.0 → mostlyai_engine-2.5.0}/pyproject.toml

@@ -1,9 +1,9 @@
 [project]
 name = "mostlyai-engine"
-version = "2.4.0"
+version = "2.5.0"
 description = "Synthetic Data Engine"
 authors = [{ name = "MOSTLY AI", email = "dev@mostly.ai" }]
-requires-python = ">=3.10"
+requires-python = ">=3.11,<3.14"
 readme = "README.md"
 license = "Apache-2.0"
 classifiers = [
@@ -14,7 +14,6 @@ classifiers = [
     "Intended Audience :: Financial and Insurance Industry",
     "Intended Audience :: Healthcare Industry",
     "Intended Audience :: Telecommunications Industry",
-    "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
@@ -33,7 +32,7 @@ dependencies = [
     "scikit-learn>=1.4.0",
     "psutil>=5.9.5,<6",  # upgrade when colab psutil is updated
     "tokenizers>=0.21.0",
-    "transformers>=4.55.0",
+    "transformers>=4.55.0,<5",  # keep <5 for vllm==0.12 compatibility
     "datasets>=3.0.0",
     "accelerate>=1.5.0",
     "peft>=0.12.0",
@@ -95,8 +94,9 @@ requires = ["hatchling", "hatch-vcs"]
 build-backend = "hatchling.build"
 
 [tool.ruff]
-target-version = "py310"
+target-version = "py311"
 line-length = 120
+extend-exclude = ["*.ipynb"]
 [tool.ruff.format]
 exclude = ["examples/*.ipynb"]
 [tool.ruff.lint]