tracebloc 1.2.78__tar.gz → 1.2.80__tar.gz

{tracebloc-1.2.78 → tracebloc-1.2.80}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tracebloc
-Version: 1.2.78
+Version: 1.2.80
 Summary: Build AI with anyone. On data that can't move. SDK for the tracebloc collaborative AI workspace.
 Author-email: tracebloc <lukas@tracebloc.io>
 License-Expression: MIT

{tracebloc-1.2.78 → tracebloc-1.2.80}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "tracebloc"
-version = "1.2.78"
+version = "1.2.80"
 description = "Build AI with anyone. On data that can't move. SDK for the tracebloc collaborative AI workspace."
 license = "MIT"
 license-files = ["license.txt"]

{tracebloc-1.2.78 → tracebloc-1.2.80}/tracebloc/convert/families/hf/__init__.py

@@ -24,7 +24,7 @@ from pathlib import Path
 from typing import Any, Mapping
 
 from tracebloc.convert.families.hf._build import head_prefixes, load, model_source
-from tracebloc.convert.families.hf._build import write_weights
+from tracebloc.convert.families.hf._build import with_pad_token_id, write_weights
 from tracebloc.convert.families.hf._model_type import _AUTO, CONFIG_FILE, _mapping
 from tracebloc.convert.families.hf._model_type import _names, model_type_of
 from tracebloc.convert.families.vision import read_normalisation
@@ -454,7 +454,8 @@ class HuggingFaceFamily:
         pooling = state["pooling"]
         if pooling and pooling != ("mean",):
             notes.append(f"declares {', '.join(pooling)} pooling; training mean-pools")
-        source = model_source(auto_class, state["config"], classes is not None, wrapper)
+        config = with_pad_token_id(state["config"], model, tokenizer)
+        source = model_source(auto_class, config, classes is not None, wrapper)
         return GeneratedPackage(
             family=self.name,
             files=(GeneratedFile(f"{stem}.py", source),),

{tracebloc-1.2.78 → tracebloc-1.2.80}/tracebloc/convert/families/hf/_build.py

@@ -202,6 +202,30 @@ def write_weights(model: Any, wrapper: str | None, target: Path) -> tuple[str, .
     return tuple(out)
 
 
+def with_pad_token_id(
+    config: Mapping[str, Any], model: Any, tokenizer: Path | None
+) -> Mapping[str, Any]:
+    """``config`` with the tokenizer's pad id, when the task class reads
+    ``config.pad_token_id`` and the checkpoint set none.
+
+    A decoder classifier with no pad id raises on any batch over 1.
+    """
+    import inspect
+
+    if config.get("pad_token_id") is not None or tokenizer is None:
+        return config
+    try:
+        reads = "pad_token_id" in inspect.getsource(type(model).forward)
+        from tokenizers import Tokenizer
+
+        padding = Tokenizer.from_file(str(tokenizer)).padding
+    except (OSError, TypeError, ValueError, ImportError):
+        return config
+    if not reads or not padding or padding.get("pad_id") is None:
+        return config
+    return {**config, "pad_token_id": padding["pad_id"]}
+
+
 def render_config(config: Mapping[str, Any]) -> str:
     """``CONFIG`` as a Python literal, one key per line."""
     lines = [f"    {key!r}: {value!r},\n" for key, value in config.items()]

{tracebloc-1.2.78 → tracebloc-1.2.80}/tracebloc/convert/families/tabular.py

@@ -1,11 +1,8 @@
 """Fitted scikit-learn, XGBoost, LightGBM, CatBoost, lifelines and scikit-survival models.
 
-A fitted estimator or ``Pipeline``, live or saved, becomes the template the
-model zoo writes by hand for its framework: ``MyModel`` builds the estimator
-from its own parameters, and the fitted model ships as the weights. A fitted
-estimator serves the tabular task of its type (classifier, regressor or
-survival model); the use case picks among the tasks that share that type.
-"""
+A fitted estimator or ``Pipeline`` becomes its framework's zoo template: ``MyModel``
+builds the estimator from its own parameters, and the fitted model ships as the
+weights. It serves the tabular task of its type: classifier, regressor or survival."""
 
 from __future__ import annotations
 
@@ -26,7 +23,9 @@ from tracebloc.convert.interfaces import (
     GeneratedPackage, Proof, Provenance, Recognition, RefusalCause, SourcedValue,
     UseCaseTask, ValueSource,
 )
+from tracebloc._framework_utils import extra_for_module
 from tracebloc.utils.constants import Framework, TaskCategory
+from tracebloc.utils.general_utils import format_exception
 
 #: The libraries whose fitted models this family reads, by top-level module.
 FRAMEWORK_OF_ROOT = {
@@ -78,6 +77,9 @@ _FUNCTIONS = frozenset(
 
 _KNOWN_ROOTS = frozenset(FRAMEWORK_OF_ROOT) | _DATA_ROOTS
 _PICKLE_SUFFIXES = (".pkl", ".pickle", ".joblib")
+_JOBLIB_LOCK = threading.Lock()
+#: Each library's own error for saved model bytes its loader cannot read.
+_LOAD_ERRORS = (("xgboost.core", "XGBoostError"), ("lightgbm.basic", "LightGBMError"), ("catboost", "CatBoostError"))
 _FROM_MODEL = Provenance(ValueSource.MODEL, "fitted estimator", verified=True)
 
 
@@ -111,11 +113,7 @@ class _Allowlisted:
         raise _UnloadableError(module, name)
 
 
-class _Unpickler(_Allowlisted, pickle.Unpickler):
-    pass
-
-
-_JOBLIB_LOCK = threading.Lock()
+class _Unpickler(_Allowlisted, pickle.Unpickler): ...
 
 
 def _load_pickled(path: Path) -> Any:
@@ -155,8 +153,21 @@ def _load(source: ConversionInput) -> Any:
             f"{path.name} names {refused} in its saved state, which training does "
             f"not load; pass the fitted model object itself instead",
         ) from None
-    except (pickle.UnpicklingError, EOFError, ValueError, IndexError, ImportError, AttributeError, TypeError):
-        return None
+    except ModuleNotFoundError as missing:
+        if missing.name not in _KNOWN_ROOTS:
+            return None  # a part of an installed library that this version lacks
+        install = f"tracebloc[{extra}]" if (extra := extra_for_module(missing.name)) else missing.name
+        message = f"{path.name} needs {missing.name}, which is not installed: pip install '{install}'"
+        raise ConversionRefusedError(RefusalCause.VERSIONS, message) from None
+    except Exception as error:
+        loaded = (getattr(sys.modules.get(m), n, None) for m, n in _LOAD_ERRORS)
+        if isinstance(error, tuple(c for c in loaded if isinstance(c, type))):
+            detail = " ".join(format_exception(error).split())[:120]
+            message = f"{path.name} holds model bytes its library cannot read ({detail})"
+            raise ConversionRefusedError(RefusalCause.NO_SOURCE, f"{message}; save it again") from None
+        if isinstance(error, (pickle.UnpicklingError, EOFError, ValueError, IndexError, ImportError, AttributeError, TypeError)):
+            return None
+        raise
 
 
 def _root(cls: type) -> str:
@@ -214,9 +225,7 @@ def _kind(model: Any, framework: Framework) -> str | None:
         return "survival"
     from sklearn.base import is_classifier, is_regressor
 
-    if is_classifier(model):
-        return "classifier"
-    return "regressor" if is_regressor(model) else None
+    return "classifier" if is_classifier(model) else "regressor" if is_regressor(model) else None
 
 
 def _feature_count(model: Any, framework: Framework) -> int | None:
@@ -374,11 +383,9 @@ class TabularFamily:
         framework, terminal = recognition.framework, _terminal(model)
         estimator = recognition.values["entry_point"].value
         _check_task(estimator, framework, kind, task.category)
-        category_origin = (
-            Provenance(ValueSource.DATASET, "use case", verified=True)
-            if task.from_use_case
-            else Provenance(ValueSource.MODEL, "estimator type")
-        )
+        category_origin = Provenance(ValueSource.DATASET, "use case", verified=True)
+        if not task.from_use_case:
+            category_origin = Provenance(ValueSource.MODEL, "estimator type")
         header = {"framework": SourcedValue(framework, _FROM_MODEL)}
         header["category"] = SourcedValue(task.category, category_origin)
         header.update(_counts(recognition, task))
@@ -430,11 +437,8 @@ def _check_task(estimator: str, framework: Framework, kind: str | None, task: Ta
 
 
 def _counts(recognition: Recognition, task: UseCaseTask) -> dict[str, SourcedValue]:
-    """The counts the header needs: the model's, held to the dataset's.
-
-    A fitted model cannot take another feature or class count, so a dataset
-    that disagrees is refused; lifelines fits whatever covariates it is given.
-    """
+    """The counts the header needs: the model's, held to the dataset's. A fitted model
+    cannot take another count; lifelines fits whatever covariates it is given."""
     from tracebloc.validation.header_rules import required_values
 
     estimator = recognition.values["entry_point"].value
@@ -470,11 +474,9 @@ def _write_weights(model: Any, path: Path, notes: list[str]) -> Path | None:
     try:
         _Unpickler(io.BytesIO(data)).load()
     except _UnloadableError as refused:
-        notes.append(
-            f"The fitted {type(_terminal(model)).__name__} is not shipped: its saved "
-            f"state names {refused}, which training does not load. Training fits it "
-            f"from the dataset."
-        )
+        name = type(_terminal(model)).__name__
+        notes.append(f"The fitted {name} is not shipped: its saved state names {refused}, "
+                     "which training does not load. Training fits it from the dataset.")
         return None
     path.write_bytes(data)
     return path
@@ -491,9 +493,7 @@ def _averaging_notes(model: Any, framework: Framework) -> list[str]:
     if framework is Framework.SKLEARN:
         from tracebloc.validation.sklearn_checks import non_averageable_estimator_reason
 
-        reason = non_averageable_estimator_reason(model)
-        if reason:
-            notes.append(reason)
+        notes.extend(filter(None, [non_averageable_estimator_reason(model)]))
     return notes
 
 

{tracebloc-1.2.78 → tracebloc-1.2.80}/tracebloc/convert/outputs.py

@@ -384,6 +384,7 @@ def torch_output_check(
                         return CheckFinding("output", False, detail)
                     ours, theirs = captured
                 compared = "the backbone's output"
+                probe.call(model)  # the head is fresh, so only this runs it
     except Exception as error:  # noqa: BLE001 - the check reports, never raises
         return CheckFinding(
             "output", False, f"comparing outputs raised {_one_line(error)}"

{tracebloc-1.2.78 → tracebloc-1.2.80}/tracebloc.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tracebloc
-Version: 1.2.78
+Version: 1.2.80
 Summary: Build AI with anyone. On data that can't move. SDK for the tracebloc collaborative AI workspace.
 Author-email: tracebloc <lukas@tracebloc.io>
 License-Expression: MIT