wizata-dsapi 2.2.0.dev24__tar.gz → 2.2.0.dev25__tar.gz

{wizata_dsapi-2.2.0.dev24 → wizata_dsapi-2.2.0.dev25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wizata_dsapi
-Version: 2.2.0.dev24
+Version: 2.2.0.dev25
 Summary: Wizata Data Science Toolkit
 Author: Wizata S.A.
 Author-email: info@wizata.com

{wizata_dsapi-2.2.0.dev24 → wizata_dsapi-2.2.0.dev25}/wizata_dsapi/__init__.py

@@ -11,7 +11,9 @@ from .execution import Execution, ExecutionStatus, ExecutionStepLog, AbortedExce
 from .execution_log import ExecutionLog
 from .experiment import Experiment
 from .ds_dataframe import DSDataFrame
-from .script import Script, ScriptConfig, ScriptType, ScriptCategory, ScriptLibrary, sanitize_blob_name, discover_from_wheel
+from .script import Script, ScriptConfig, ScriptType, ScriptCategory, sanitize_blob_name
+from .python_package import (PythonPackage, PythonPackageVersion, PythonPackageSource, PythonPackagePin,
+                             normalize_python_package_name)
 from .template import Template, TemplateProperty
 from .solution_component import SolutionComponent, SolutionType
 from .business_label import BusinessLabel

{wizata_dsapi-2.2.0.dev24 → wizata_dsapi-2.2.0.dev25}/wizata_dsapi/context.py

@@ -77,6 +77,9 @@ class Context(ILogger):
         # version — not the Postgres "current" one. Left None off the git path.
         self.branch = None
         self.sha = None
+        # DEV-6120: the Python packages mounted for the run, [{name, version, sha256}];
+        # a model trained in it records them.
+        self.python_packages = []
         self.template = template
         self.registration = registration
         self.now = now

{wizata_dsapi-2.2.0.dev24 → wizata_dsapi-2.2.0.dev25}/wizata_dsapi/pipeline.py

@@ -14,6 +14,7 @@ from .twin import Twin
 from .twinregistration import TwinRegistration, TwinRegistrationProperty
 from .datapoint import DataPoint
 from .formula import normalize_formulas
+from .python_package import PythonPackagePin, normalize_python_package_name
 
 from enum import Enum
 
@@ -592,6 +593,9 @@ class Pipeline(ApiDto):
     :ivar list steps: list of steps connected between by their inputs and outputs.
     :ivar uuid.UUID template_id: template id associated to the pipeline.
     :ivar dict variables: dictionary of variable name with they wizata_dsapi.VarType.
+    :ivar list python_packages: the Python packages the pipeline imports, as
+        ``PythonPackagePin`` (JSON ``pythonPackages``): an uploaded package with its version,
+        a git package by name only.
     :ivar uuid.UUID createdById: unique identifier of creating user.
     :ivar int createdDate: timestamp of created date.
     :ivar uuid.UUID updatedById: unique identifier of updating user.
@@ -615,7 +619,8 @@ class Pipeline(ApiDto):
                  steps: list = None,
                  plots: list = None,
                  template_id: uuid.UUID = None,
-                 experiment_id: uuid.UUID = None):
+                 experiment_id: uuid.UUID = None,
+                 python_packages: list = None):
 
         if pipeline_id is None:
             pipeline_id = uuid.uuid4()
@@ -637,6 +642,10 @@ class Pipeline(ApiDto):
         self.plots = plots
         if plots is None:
             self.plots = []
+        self._python_packages = list(python_packages) if python_packages is not None else []
+        # DEV-6120: whether the list was given (constructor, JSON key, assignment). A save
+        # that leaves it out keeps the stored list - the 12.2 editor does not know it.
+        self._python_packages_given = python_packages is not None
         if steps is not None:
             for step in steps:
                 if not isinstance(step, PipelineStep):
@@ -663,11 +672,75 @@ class Pipeline(ApiDto):
     def key(self):
         del self._key
 
+    @property
+    def python_packages(self) -> list:
+        """the Python packages the pipeline imports, as ``PythonPackagePin`` (DEV-6120)."""
+        return self._python_packages
+
+    @python_packages.setter
+    def python_packages(self, value):
+        self._python_packages = list(value) if value is not None else []
+        self._python_packages_given = True
+
+    @property
+    def python_packages_given(self) -> bool:
+        """
+        whether ``python_packages`` was set (constructor, ``pythonPackages`` in the JSON, or
+        an assignment). An update that leaves it out keeps the stored list; ``[]`` clears it.
+        """
+        return self._python_packages_given
+
+    def add_python_package(self, name: str, version: str = None) -> PythonPackagePin:
+        """
+        import a Python package in this pipeline (DEV-6120). Adding a package already listed
+        replaces its pin.
+        :param name: the package's name.
+        :param version: the uploaded version to run with; None for a git package.
+        """
+        pin = PythonPackagePin(name=name, version=version)
+        normalized = normalize_python_package_name(name)
+        self.python_packages = [p for p in self.python_packages
+                                if not isinstance(p.name, str)
+                                or normalize_python_package_name(p.name) != normalized]
+        self.python_packages.append(pin)
+        return pin
+
+    def check_python_packages(self) -> bool:
+        """
+        validate the shape of ``python_packages``: a name each, a version that is text when
+        given, no package twice. Whether the package and version exist is checked on save.
+        """
+        seen = set()
+        for pin in self.python_packages:
+            if not isinstance(pin, PythonPackagePin):
+                raise ValueError(f"pythonPackages entries must be PythonPackagePin, not {type(pin).__name__}")
+            if not isinstance(pin.name, str) or not pin.name:
+                raise ValueError(f"pythonPackages: each entry needs a package name, got {pin.name!r}")
+            if pin.version is not None and (not isinstance(pin.version, str) or not pin.version):
+                raise ValueError(f"pythonPackages: the version of {pin.name!r} must be text, "
+                                 f"got {pin.version!r}")
+            normalized = normalize_python_package_name(pin.name)
+            if normalized in seen:
+                raise ValueError(f"pythonPackages: package {pin.name!r} is listed twice")
+            seen.add(normalized)
+        return True
+
+    @staticmethod
+    def _read_python_packages(value) -> list:
+        if value is None or value == "":
+            return []
+        if isinstance(value, str):
+            value = json.loads(value)
+        if not isinstance(value, list) or not all(isinstance(v, dict) for v in value):
+            raise ValueError('pythonPackages must be a list of {"name": ..., "version": ...}')
+        return [PythonPackagePin.from_dict(v) for v in value]
+
     def check_path(self) -> bool:
         """
         validate that steps create a valid path.
         return true if path is valid, otherwise raise errors
         """
+        self.check_python_packages()
 
         followed = []  # all steps already followed
         produced = []  # all outputs already produced
@@ -1022,6 +1095,8 @@ class Pipeline(ApiDto):
                 self.plots = json.loads(obj["plots"])
             else:
                 self.plots = obj["plots"]
+        if "pythonPackages" in obj.keys():
+            self.python_packages = self._read_python_packages(obj["pythonPackages"])
         if "createdById" in obj.keys() and obj["createdById"] is not None:
             self.createdById = obj["createdById"]
         if "createdDate" in obj.keys() and obj["createdDate"] is not None:
@@ -1060,6 +1135,15 @@ class Pipeline(ApiDto):
         if self.variables is not None:
             self.check_variables()
             obj["variables"] = dict(self.variables) if as_git else json.dumps(self.variables)
+        # DEV-6120. In git only when not empty, so pipeline files that import nothing do not
+        # change. Elsewhere always, as text like steps / plots / variables (the C# column):
+        # an update that leaves the key out keeps the stored list, so "[]" is how it clears.
+        pins = [pin.to_dict() for pin in self.python_packages]
+        if as_git:
+            if pins:
+                obj["pythonPackages"] = pins
+        else:
+            obj["pythonPackages"] = json.dumps(pins)
         if self.template_id is not None:
             obj["templateId"] = str(self.template_id)
         if self.experiment_id is not None:

{wizata_dsapi-2.2.0.dev24 → wizata_dsapi-2.2.0.dev25}/wizata_dsapi/pipeline_image.py

@@ -1,3 +1,4 @@
+import contextlib
 import json
 import os
 import wizata_dsapi
@@ -39,6 +40,10 @@ class PipelineImage(ApiDto):
         self.datapoints = {}
         self.scripts = []
         self.models = {}
+        # DEV-6120: the Python packages the pipeline pins, bundled at build time
+        # (python_packages.json + python_packages/<sha256>.whl), mounted by the loader.
+        self.python_package_index = []
+        self.python_package_files = {}
 
     @classmethod
     def route(cls):
@@ -102,26 +107,32 @@ class PipelineImage(ApiDto):
     @classmethod
     def load(cls,
              pipeline_image_id: str = None,
-             g_stream=None) -> 'PipelineImage':
+             g_stream=None,
+             mount=None) -> 'PipelineImage':
         """
         deserialize from a stream (e.g. file handle).
         :param pipeline_image_id: id of the image.
         :param g_stream: readable stream.
+        :param mount: see :meth:`loads`.
         :return: PipelineImage
         """
         if g_stream is None:
             raise ValueError("No stream provided")
         g_bytes = g_stream.read()
-        return cls.loads(pipeline_image_id=pipeline_image_id, g_bytes=g_bytes)
+        return cls.loads(pipeline_image_id=pipeline_image_id, g_bytes=g_bytes, mount=mount)
 
     @classmethod
     def loads(cls,
               pipeline_image_id: str = None,
-              g_bytes=None) -> 'PipelineImage':
+              g_bytes=None,
+              mount=None) -> 'PipelineImage':
         """
         deserialize from a bytes array an image.
         :param pipeline_image_id: id of the image.
         :param g_bytes: from loaded content as bytes.
+        :param mount: called with the image once its pipeline and Python packages are read;
+            returns the context manager the models are unpickled in (DEV-6120: the packages
+            they were pickled against made importable). None: no packages are mounted.
         :return: PipelineImage
         """
         # unpack the content
@@ -142,6 +153,13 @@ class PipelineImage(ApiDto):
             pipeline._load_related_entities(pipeline_dict)
             image.pipeline = pipeline
 
+            # DEV-6120: the Python packages the pipeline pins, as bundled at build time
+            if "python_packages.json" in g_files:
+                image.python_package_index = json.loads(g_files["python_packages.json"])
+                image.python_package_files = {
+                    k[len("python_packages/"):].rsplit(".", 1)[0]: v for k, v in g_files.items()
+                    if k.startswith("python_packages/") and k.endswith((".whl", ".zip"))}
+
             # load datapoints if present
             if "datapoints.json" in g_files:
                 datapoints_json = json.loads(g_files['datapoints.json'])
@@ -177,34 +195,39 @@ class PipelineImage(ApiDto):
                 if json_file not in json_files:
                     raise KeyError(f"missing json file for script pkl {pkl_file}")
 
-            # process the models
-            model_files = {k: v for k, v in g_files.items() if k.startswith('models/')}
-            for pkl_file in model_files:
-                parts = pkl_file.split('/')
-                if len(parts) == 2:
-                    identifier = os.path.splitext(parts[1])[0]
-                else:
-                    identifier = parts[1]
-
-                if identifier not in image.models:
-                    key, twin_hardware_id, property_value, alias = wizata_dsapi.ModelInfo.split_identifier(identifier)
-                    image.models[identifier] = wizata_dsapi.ModelInfo(
-                        key=key,
-                        twin_hardware_id=twin_hardware_id,
-                        property_value=property_value,
-                        alias=alias
-                    )
-                if len(parts) == 2:
-                    image.models[identifier].file_format = os.path.splitext(parts[1])[1].lstrip('.')
-                    image.models[identifier].uncompress_bytes_to_model(g_files[pkl_file])
-                if len(parts) == 3:
-                    image.models[identifier].add_file(wizata_dsapi.ModelFile(
-                        full_path=pkl_file,
-                        path=parts[2],
-                        content=g_files[pkl_file]
-                    ))
+            # process the models, in the packages they were pickled against (DEV-6120)
+            with (mount(image) if mount is not None else contextlib.nullcontext()):
+                cls._load_models(image, g_files)
 
             return image
 
         except Exception as e:
             raise Exception(f'invalid image - {e}')
+
+    @staticmethod
+    def _load_models(image: 'PipelineImage', g_files: dict) -> None:
+        model_files = {k: v for k, v in g_files.items() if k.startswith('models/')}
+        for pkl_file in model_files:
+            parts = pkl_file.split('/')
+            if len(parts) == 2:
+                identifier = os.path.splitext(parts[1])[0]
+            else:
+                identifier = parts[1]
+
+            if identifier not in image.models:
+                key, twin_hardware_id, property_value, alias = wizata_dsapi.ModelInfo.split_identifier(identifier)
+                image.models[identifier] = wizata_dsapi.ModelInfo(
+                    key=key,
+                    twin_hardware_id=twin_hardware_id,
+                    property_value=property_value,
+                    alias=alias
+                )
+            if len(parts) == 2:
+                image.models[identifier].file_format = os.path.splitext(parts[1])[1].lstrip('.')
+                image.models[identifier].uncompress_bytes_to_model(g_files[pkl_file])
+            if len(parts) == 3:
+                image.models[identifier].add_file(wizata_dsapi.ModelFile(
+                    full_path=pkl_file,
+                    path=parts[2],
+                    content=g_files[pkl_file]
+                ))

wizata_dsapi-2.2.0.dev25/wizata_dsapi/python_package.py

@@ -0,0 +1,222 @@
+"""
+Python packages a pipeline imports (DEV-6120, epic DEV-6119).
+
+A customer's own code, typically scikit-learn classes in an installable package, made
+importable where a pipeline runs (runners, pipeline images, edges) under its original
+dotted name, so models pickled against it load unchanged.
+
+PythonPackage — one package, named as pip names it: the name is normalised the pip way
+  (``XpertRef_Tools`` is ``xpertref-tools``). ``source`` says where its code comes from:
+
+  - ``upload``: pure-Python wheels uploaded one version at a time (option A);
+  - ``git``: the ``packages/<name>/`` folder of the attached repository (option B), whose
+    version is the commit a pipeline is read at.
+
+  ``import_names`` are the top-level modules it provides (``xpertref``). ``versions`` is
+  filled on read only.
+
+PythonPackageVersion — one uploaded wheel of an ``upload`` package. Immutable: a version is
+  never replaced, a fix is a new version.
+
+The terms are pip's and PyPI's (name, version). "Release" is not used: Wizata already has
+release branches and edge releases. This replaces the script "library" concept
+(``ScriptLibrary``, ``ScriptType.MODULE``), which was never used and is removed.
+"""
+import re
+from datetime import datetime, timezone
+from enum import Enum
+from typing import List, Optional
+
+
+def normalize_python_package_name(name: str) -> str:
+    """
+    A package name as pip compares it (PEP 503): ``XpertRef_Tools.v2`` -> ``xpertref-tools-v2``.
+    Validating it is the package client's job.
+    """
+    return re.sub(r"[-_.]+", "-", name).lower()
+
+
+class PythonPackageSource(Enum):
+    """Where a Python package's code comes from."""
+    UPLOAD = "upload"
+    GIT = "git"
+
+
+def _ts_out(value):
+    if value is None:
+        return None
+    if value.tzinfo is None:
+        value = value.replace(tzinfo=timezone.utc)
+    return value.isoformat()
+
+
+def _ts_in(value):
+    if value is None or isinstance(value, datetime):
+        return value
+    return datetime.fromisoformat(str(value).replace("Z", "+00:00"))
+
+
+def _first(data: dict, *keys):
+    for key in keys:
+        if data.get(key) is not None:
+            return data[key]
+    return None
+
+
+class PythonPackageVersion:
+    """
+    One uploaded wheel, stored in PostgreSQL ``"PythonPackageVersion"``. PK (``name``,
+    ``version``).
+
+    :ivar str name: the package's normalised name.
+    :ivar str version: the version, normalised as pip does (PEP 440).
+    :ivar str filename: the wheel's file name.
+    :ivar str sha256: hex digest of the wheel.
+    :ivar int size: the wheel's size in bytes.
+    :ivar list requires_dist: the wheel's ``Requires-Dist`` lines.
+    :ivar str requires_python: the wheel's ``Requires-Python``.
+    :ivar str uploaded_by: the user who uploaded it.
+    :ivar datetime uploaded_at: when it was uploaded.
+    """
+
+    def __init__(self, name: str = None, version: str = None, filename: str = None,
+                 sha256: str = None, size: int = None, requires_dist: list = None,
+                 requires_python: str = None):
+        self.name: Optional[str] = name
+        self.version: Optional[str] = version
+        self.filename: Optional[str] = filename
+        self.sha256: Optional[str] = sha256
+        self.size: Optional[int] = size
+        self.requires_dist: Optional[List[str]] = requires_dist
+        self.requires_python: Optional[str] = requires_python
+        self.uploaded_by: Optional[str] = None
+        self.uploaded_at: Optional[datetime] = None
+
+    def api_id(self) -> str:
+        return f"{self.name}=={self.version}"
+
+    def to_dict(self) -> dict:
+        obj = {}
+        for key, value in (("name", self.name), ("version", self.version),
+                           ("filename", self.filename), ("sha256", self.sha256),
+                           ("size", self.size), ("requiresDist", self.requires_dist),
+                           ("requiresPython", self.requires_python),
+                           ("uploadedBy", self.uploaded_by),
+                           ("uploadedAt", _ts_out(self.uploaded_at))):
+            if value is not None:
+                obj[key] = value
+        return obj
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "PythonPackageVersion":
+        obj = cls(name=data.get("name"), version=data.get("version"),
+                  filename=data.get("filename"), sha256=data.get("sha256"),
+                  size=data.get("size"),
+                  requires_dist=_first(data, "requiresDist", "requires_dist"),
+                  requires_python=_first(data, "requiresPython", "requires_python"))
+        obj.uploaded_by = _first(data, "uploadedBy", "uploaded_by")
+        obj.uploaded_at = _ts_in(_first(data, "uploadedAt", "uploaded_at"))
+        return obj
+
+    def __repr__(self) -> str:
+        return f"PythonPackageVersion({self.name}=={self.version})"
+
+
+class PythonPackage:
+    """
+    A Python package, stored in PostgreSQL ``"PythonPackage"``. PK ``name``.
+
+    :ivar str name: the name, normalised as pip does.
+    :ivar PythonPackageSource source: upload or git; never changes.
+    :ivar list import_names: the top-level modules it provides.
+    :ivar str description: free text.
+    :ivar str created_by: the user who created it.
+    :ivar datetime created_at: when it was created.
+    :ivar datetime updated_at: when it last changed.
+    :ivar str problem: why it cannot be used, when it cannot (a git folder that does not
+        compile, a name clash with the repository); set by the platform.
+    :ivar list versions: its uploaded versions, newest first (filled on read only).
+    """
+
+    @classmethod
+    def route(cls):
+        return "pythonpackages"
+
+    def __init__(self, name: str = None, source: PythonPackageSource = None,
+                 import_names: list = None, description: str = None):
+        self.name: Optional[str] = name
+        self.source: Optional[PythonPackageSource] = source
+        self.import_names: List[str] = import_names if import_names is not None else []
+        self.description: Optional[str] = description
+        self.created_by: Optional[str] = None
+        self.created_at: Optional[datetime] = None
+        self.updated_at: Optional[datetime] = None
+        self.problem: Optional[str] = None
+        self.versions: Optional[List[PythonPackageVersion]] = None
+
+    def api_id(self) -> str:
+        return self.name
+
+    def to_dict(self) -> dict:
+        obj = {}
+        if self.name is not None:
+            obj["name"] = self.name
+        if self.source is not None:
+            obj["source"] = self.source.value
+        obj["importNames"] = list(self.import_names) if isinstance(self.import_names, list) \
+            else self.import_names
+        for key, value in (("description", self.description), ("createdBy", self.created_by),
+                           ("createdAt", _ts_out(self.created_at)),
+                           ("updatedAt", _ts_out(self.updated_at)), ("problem", self.problem)):
+            if value is not None:
+                obj[key] = value
+        if self.versions is not None:
+            obj["versions"] = [v.to_dict() for v in self.versions]
+        return obj
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "PythonPackage":
+        source = data.get("source")
+        obj = cls(name=data.get("name"),
+                  source=PythonPackageSource(source) if source is not None else None,
+                  import_names=_first(data, "importNames", "import_names"),
+                  description=data.get("description"))
+        obj.created_by = _first(data, "createdBy", "created_by")
+        obj.created_at = _ts_in(_first(data, "createdAt", "created_at"))
+        obj.updated_at = _ts_in(_first(data, "updatedAt", "updated_at"))
+        obj.problem = data.get("problem")
+        if data.get("versions") is not None:
+            obj.versions = [PythonPackageVersion.from_dict(v) for v in data["versions"]]
+        return obj
+
+    def __repr__(self) -> str:
+        source = self.source.value if self.source is not None else None
+        return f"PythonPackage(name={self.name!r}, source={source!r})"
+
+
+class PythonPackagePin:
+    """
+    One entry of a pipeline's ``pythonPackages`` list: a package the pipeline imports.
+
+    :ivar str name: the package's name.
+    :ivar str version: the uploaded version it runs with; None for a git package, whose
+        code is read at the commit the pipeline is read at.
+    """
+
+    def __init__(self, name: str = None, version: str = None):
+        self.name: Optional[str] = name
+        self.version: Optional[str] = version
+
+    def to_dict(self) -> dict:
+        obj = {"name": self.name}
+        if self.version is not None:
+            obj["version"] = self.version
+        return obj
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "PythonPackagePin":
+        return cls(name=data.get("name"), version=data.get("version"))
+
+    def __repr__(self) -> str:
+        return f"PythonPackagePin({self.name}=={self.version})" if self.version \
+            else f"PythonPackagePin({self.name})"

{wizata_dsapi-2.2.0.dev24 → wizata_dsapi-2.2.0.dev25}/wizata_dsapi/script.py

@@ -16,12 +16,13 @@ class ScriptType(Enum):
     defines the type of script execution.
         - "wizata" builtin wizata library function (wizata. prefix)
         - "api" dill-serialized script stored on backend (DB + blob .pkl)
-        - "module" function from an installed custom python module
         - "raw" raw .py source stored on blob, compiled at runtime
+
+    A customer's own package is not a script type: a pipeline imports it as a Python
+    package (``PythonPackage``, DEV-6120).
     """
     WIZATA = 'wizata'
     API = 'api'
-    MODULE = 'module'
     RAW = 'raw'
 
 
@@ -94,12 +95,10 @@ class ScriptConfig(ApiDto):
 
     def __init__(self,
                  function=None,
-                 library=None,
                  properties_mapping=None,
                  properties: dict = None,
                  script_type=None):
         self.function = function
-        self.library = library
         self.properties = properties
         self.properties_mapping = properties_mapping
         self.script_type = script_type
@@ -111,9 +110,7 @@ class ScriptConfig(ApiDto):
         else:
             if "function" in obj:
                 self.function = obj["function"]
-            if "library" in obj:
-                self.library = obj["library"]
-            elif "script" in obj:
+            if "script" in obj:
                 self.function = obj["script"]
             if "properties" in obj:
                 self.properties = obj["properties"]
@@ -126,8 +123,6 @@ class ScriptConfig(ApiDto):
         obj = {
             "function": self.function
         }
-        if self.library is not None:
-            obj["library"] = self.library
         if self.properties is not None and isinstance(self.properties, dict):
             obj["properties"] = self.properties
         if self.properties_mapping is not None and isinstance(self.properties_mapping, dict):
@@ -139,8 +134,7 @@ class ScriptConfig(ApiDto):
     def infer_script_type(self):
         """
         Return explicit script_type if set, otherwise infer from convention:
-        - function starts with "wizata." -> LIBRARY_WIZATA
-        - library field is set -> LIBRARY_CUSTOM
+        - function starts with "wizata." -> WIZATA (built-in)
         - else -> API
         Note: RAW cannot be inferred; must be set explicitly.
         """
@@ -148,8 +142,6 @@ class ScriptConfig(ApiDto):
             return self.script_type
         if self.function is not None and str(self.function).startswith("wizata."):
             return ScriptType.WIZATA
-        if self.library is not None:
-            return ScriptType.MODULE
         return ScriptType.API
 
 
@@ -160,10 +152,8 @@ class Script(ApiDto):
     :ivar uuid.UUID script_id: technical id of the script.
     :ivar str name: unique name of the script function (unique key within its type).
     :ivar str description: human-readable description of what the script does.
-    :ivar ScriptType script_type: type of script execution (api, raw, module, wizata).
+    :ivar ScriptType script_type: type of script execution (api, raw, wizata).
     :ivar str version: version string of the script.
-    :ivar str library: key of the ScriptLibrary this script belongs to (module type only).
-    :ivar str module: Python import path for import_module() (module type only, e.g. 'my_lib.transforms').
     :ivar callable function: the function as a callable object (not serialized).
     :ivar str source: python source code of the script (not serialized).
     """
@@ -187,7 +177,7 @@ class Script(ApiDto):
         return "json"
 
     def __init__(self, script_id=None, description=None, function=None,
-                 script_type=None, version=None, library=None, category=None,
+                 script_type=None, version=None, category=None,
                  properties=None):
 
         # Id
@@ -200,8 +190,6 @@ class Script(ApiDto):
         self.description = description
         self.script_type = script_type
         self.version = version
-        self.library = library
-        self.module = None
         self.category = category
         self.properties = properties if properties is not None else {}
 
@@ -265,10 +253,6 @@ class Script(ApiDto):
             obj["type"] = self.script_type.value
         if self.version is not None:
             obj["version"] = self.version
-        if self.library is not None:
-            obj["library"] = self.library
-        if self.module is not None:
-            obj["module"] = self.module
         if self.source is not None:
             obj["source"] = self.source
         if self.category is not None:
@@ -300,10 +284,6 @@ class Script(ApiDto):
             self.script_type = ScriptType(obj["type"])
         if "version" in obj:
             self.version = obj["version"]
-        if "library" in obj:
-            self.library = obj["library"]
-        if "module" in obj:
-            self.module = obj["module"]
         if "source" in obj and obj["source"] is not None:
             self.source = obj["source"]
         if "category" in obj and obj["category"] is not None:
@@ -404,124 +384,6 @@ class Script(ApiDto):
                 self.source = source_code
 
 
-class ScriptLibrary:
-    """
-    Represents a custom library registration.
-    Describes an installed Python package that exposes functions usable in pipelines.
-
-    Metadata is stored in PostgreSQL; function entries are Script rows with
-    script_type=MODULE and a foreign key back to this library's key.
-
-    :ivar str key: unique identifier/key for this library registration.
-    :ivar str package_name: pip package name (e.g. 'my-custom-lib').
-    :ivar str module_name: Python import module name (e.g. 'my_custom_lib').
-    :ivar str version: version string of the registered library.
-    :ivar str description: human-readable description of the library.
-    :ivar list functions: list of Script entries (script_type=MODULE).
-    """
-
-    def __init__(self,
-                 key: str = None,
-                 package_name: str = None,
-                 module_name: str = None,
-                 version: str = None,
-                 description: str = None,
-                 functions: list = None):
-        self.key = key
-        self.package_name = package_name
-        self.module_name = module_name
-        self.version = version
-        self.description = description
-        if functions is None:
-            functions = []
-        self.functions = functions
-
-    @classmethod
-    def from_dict(cls, data: dict):
-        obj = ScriptLibrary()
-        obj.key = data.get("key")
-        obj.package_name = data.get("packageName") or data.get("package_name")
-        obj.module_name = data.get("moduleName") or data.get("module_name")
-        obj.version = data.get("version")
-        obj.description = data.get("description")
-        if "functions" in data and data["functions"] is not None:
-            for func_data in data["functions"]:
-                script = Script.from_dict(func_data)
-                script.script_type = ScriptType.MODULE
-                script.library = obj.key
-                # "module" field = import path; falls back to library module_name
-                if script.module is None:
-                    script.module = obj.module_name
-                obj.functions.append(script)
-        return obj
-
-    def to_dict(self) -> dict:
-        obj = {}
-        if self.key is not None:
-            obj["key"] = self.key
-        if self.package_name is not None:
-            obj["packageName"] = self.package_name
-        if self.module_name is not None:
-            obj["moduleName"] = self.module_name
-        if self.version is not None:
-            obj["version"] = self.version
-        if self.description is not None:
-            obj["description"] = self.description
-        if self.functions is not None and len(self.functions) > 0:
-            obj["functions"] = [f.to_dict() for f in self.functions]
-        return obj
-
-    def get_function(self, name: str):
-        """
-        Find a function entry by name.
-        :param name: function name to search.
-        :return: Script or None.
-        """
-        for func in self.functions:
-            if func.name == name:
-                return func
-        return None
-
-
-def discover_from_wheel(wheel_source) -> ScriptLibrary:
-    """
-    Extract wizata.json manifest from a .whl file and parse into ScriptLibrary.
-
-    The manifest should be placed inside the top-level package directory
-    of the wheel (e.g. ``my_custom_lib/wizata.json``).
-
-    :param wheel_source: filesystem path (str) or bytes content of the .whl file.
-    :return: ScriptLibrary parsed from the manifest.
-    :raises FileNotFoundError: if no wizata.json found in the wheel.
-    :raises ValueError: if wizata.json is missing the required 'key' field.
-    """
-    import zipfile
-    import json
-    import io
-
-    if isinstance(wheel_source, bytes):
-        file_obj = io.BytesIO(wheel_source)
-    else:
-        file_obj = wheel_source
-
-    with zipfile.ZipFile(file_obj, 'r') as zf:
-        candidates = [n for n in zf.namelist()
-                      if n.endswith('/wizata.json') or n == 'wizata.json']
-        if not candidates:
-            raise FileNotFoundError(
-                f"no wizata.json found in wheel"
-            )
-        # prefer the shortest path (top-level package)
-        candidates.sort(key=len)
-        raw = zf.read(candidates[0])
-
-    data = json.loads(raw)
-    if "key" not in data:
-        raise ValueError("wizata.json must contain a 'key' field")
-
-    return ScriptLibrary.from_dict(data)
-
-
 class Function:
     """
     Python Code Function

wizata_dsapi-2.2.0.dev25/wizata_dsapi/version.py

@@ -0,0 +1 @@
+__version__ = "2.2.0.dev25"

{wizata_dsapi-2.2.0.dev24 → wizata_dsapi-2.2.0.dev25}/wizata_dsapi/wizata_dsapi_client.py

@@ -30,6 +30,7 @@ from .plot import Plot
 from .request import Request
 from .formula import normalize_formulas
 from .mlmodel import ModelInfo, ModelList, ModelFile
+from .python_package import PythonPackage, PythonPackageVersion, PythonPackagePin
 from .experiment import Experiment
 from .script import Script
 from .execution import Execution, ExecutionStatus
@@ -661,7 +662,7 @@ class WizataDSAPIClient(ApiInterface, ApiDtoInterface):
         Download a script with its artifacts (function/source) as a dill binary.
 
         :param name: script name.
-        :param script_type: one of 'api', 'raw', 'module' (default: 'api').
+        :param script_type: one of 'api', 'raw' (default: 'api').
         :return: Script with function loaded.
         """
         params = {'type': script_type}
@@ -1375,7 +1376,8 @@ class WizataDSAPIClient(ApiInterface, ApiDtoInterface):
 
     def upload_model(self,
                      model_info: ModelInfo,
-                     bytes_content = None):
+                     bytes_content = None,
+                     python_packages: list = None):
         """
         upload a model within the model repository.
             - by default use model_info.trained_model and convert it to a pickle
@@ -1383,9 +1385,22 @@ class WizataDSAPIClient(ApiInterface, ApiDtoInterface):
             - model_info.file_format must be set properly to 'pkl' or 'pt'
         :param model_info: model info, with at least key (+twin, +property, +alias) and trained_model.
         :param bytes_content: bytes[] of your torch or pickle model.
+        :param python_packages: the Python packages (uploaded with upload_python_package) the
+            model was trained with, each with its version: PythonPackagePin or
+            {"name": ..., "version": ...}. A deploy then warns when the pipeline pins
+            another version (DEV-6120).
         """
         if model_info.trained_model is None and bytes_content is None:
             raise ValueError("model_info must have a trained model (to pickle) or bytes content")
+        if python_packages is not None:
+            declared = []
+            for pin in python_packages:
+                entry = pin.to_dict() if isinstance(pin, PythonPackagePin) else dict(pin)
+                if not entry.get("name") or not entry.get("version"):
+                    raise ValueError(f"python_packages: each needs a name and the version the model "
+                                     f"was trained with, got {entry!r}")
+                declared.append({"name": entry["name"], "version": str(entry["version"])})
+            model_info.metadata = {**(model_info.metadata or {}), "pythonPackages": declared}
         if bytes_content is None:
             bytes_content = pickle.dumps(model_info.trained_model)
         files = {
@@ -1429,6 +1444,92 @@ class WizataDSAPIClient(ApiInterface, ApiDtoInterface):
         else:
             raise self.__raise_error(response)
 
+    # ------------------------------------------------------------------ Python packages (DEV-6121)
+
+    @staticmethod
+    def _quote(value) -> str:
+        return urllib.parse.quote(str(value), safe="")
+
+    def upload_python_package(self, wheel, filename: str = None,
+                              description: str = None) -> PythonPackageVersion:
+        """
+        Upload a version of a Python package your pipelines import: the pure-Python wheel
+        ``python -m build --wheel`` makes (e.g. ``dist/xpertref-1.4.0-py3-none-any.whl``).
+        Its name, version and requirements come from the wheel; the package is created by
+        its first upload. A version is immutable: a fix is a new version.
+
+        :param wheel: path to the .whl file, or its bytes (then give ``filename``).
+        :param filename: the wheel's file name, when ``wheel`` is bytes.
+        :param description: the package's description, on its first upload.
+        :return: the uploaded PythonPackageVersion.
+        """
+        if isinstance(wheel, (bytes, bytearray)):
+            if not filename:
+                raise ValueError("give the wheel's file name with its bytes "
+                                 "(e.g. xpertref-1.4.0-py3-none-any.whl)")
+            data = bytes(wheel)
+        else:
+            path = str(wheel)
+            filename = filename or os.path.basename(path)
+            with open(path, "rb") as f:
+                data = f.read()
+        headers = self.__header()
+        headers.pop("Content-Type", None)
+        response = requests.post(self.__url() + "pythonpackages/",
+                                 headers=headers,
+                                 data={"description": description} if description else None,
+                                 files={"file": (filename, data, "application/octet-stream")})
+        if response.status_code != 201:
+            raise self.__raise_error(response)
+        return PythonPackageVersion.from_dict(response.json())
+
+    def search_python_packages(self, search_query=None) -> PagedQueryResult:
+        """
+        :param search_query: SearchQuery or dict; filters on ``name`` and ``source``
+            (upload / git), ``_search`` over name and description.
+        :return: PagedQueryResult with PythonPackage results.
+        """
+        if search_query is None:
+            search_query = SearchQuery()
+        elif isinstance(search_query, dict):
+            search_query = SearchQuery.from_dict(search_query)
+        response = self._planner_request("POST", "pythonpackages/search", body=search_query.to_dict())
+        return PagedQueryResult.from_dict(response.json(), dto_class=PythonPackage)
+
+    def get_python_package(self, name: str):
+        """
+        :return: the PythonPackage with its versions (newest first), or None.
+        """
+        response = self._planner_request("GET", f"pythonpackages/{self._quote(name)}/",
+                                         expected=(200, 404))
+        return PythonPackage.from_dict(response.json()) if response.status_code == 200 else None
+
+    def download_python_package(self, name: str, version: str, path: str = None):
+        """
+        :param path: where to write the wheel; None returns its bytes.
+        :return: the wheel's bytes, or ``path``.
+        """
+        response = requests.get(
+            self.__url() + f"pythonpackages/{self._quote(name)}/versions/{self._quote(version)}/download",
+            headers=self.__header())
+        if response.status_code != 200:
+            raise self.__raise_error(response)
+        if path is None:
+            return response.content
+        with open(path, "wb") as f:
+            f.write(response.content)
+        return path
+
+    def delete_python_package_version(self, name: str, version: str):
+        """Refused (409) while a stored pipeline pins it."""
+        self._planner_request("DELETE",
+                              f"pythonpackages/{self._quote(name)}/versions/{self._quote(version)}/",
+                              expected=(204,))
+
+    def delete_python_package(self, name: str):
+        """Refused (409) while it has versions, or a stored pipeline imports it."""
+        self._planner_request("DELETE", f"pythonpackages/{self._quote(name)}/", expected=(204,))
+
     def upsert_experiment(self, key: str, name: str, pipeline=None, twin=None):
         """
         Upsert an experiment.

{wizata_dsapi-2.2.0.dev24 → wizata_dsapi-2.2.0.dev25}/wizata_dsapi.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wizata_dsapi
-Version: 2.2.0.dev24
+Version: 2.2.0.dev25
 Summary: Wizata Data Science Toolkit
 Author: Wizata S.A.
 Author-email: info@wizata.com

{wizata_dsapi-2.2.0.dev24 → wizata_dsapi-2.2.0.dev25}/wizata_dsapi.egg-info/SOURCES.txt

@@ -42,6 +42,7 @@ wizata_dsapi/pipeline.py
 wizata_dsapi/pipeline_image.py
 wizata_dsapi/plot.py
 wizata_dsapi/product.py
+wizata_dsapi/python_package.py
 wizata_dsapi/request.py
 wizata_dsapi/script.py
 wizata_dsapi/search.py

wizata_dsapi-2.2.0.dev24/wizata_dsapi/version.py

@@ -1 +0,0 @@
-__version__ = "2.2.0.dev24"