dtlpy 1.118.15__tar.gz → 1.119.5__tar.gz

{dtlpy-1.118.15 → dtlpy-1.119.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dtlpy
-Version: 1.118.15
+Version: 1.119.5
 Summary: SDK and CLI for Dataloop platform
 Home-page: https://github.com/dataloop-ai/dtlpy
 Author: Dataloop Team

dtlpy-1.119.5/dtlpy/__version__.py

@@ -0,0 +1 @@
+version = '1.119.5'

{dtlpy-1.118.15 → dtlpy-1.119.5}/dtlpy/entities/app.py

@@ -48,6 +48,7 @@ class App(entities.BaseEntity):
     metadata = attr.ib(type=dict)
     status = attr.ib(type=entities.CompositionStatus)
     settings = attr.ib(type=dict)
+    dpk = attr.ib(type=entities.Dpk)
 
     # sdk
     _project = attr.ib(type=entities.Project, repr=False)
@@ -57,11 +58,12 @@ class App(entities.BaseEntity):
 
     @_repositories.default
     def set_repositories(self):
-        reps = namedtuple('repositories', field_names=['projects', 'apps', 'compositions'])
+        reps = namedtuple('repositories', field_names=['projects', 'apps', 'compositions', 'models'])
         return reps(
             projects=repositories.Projects(client_api=self._client_api),
             apps=repositories.Apps(client_api=self._client_api, project=self._project, project_id=self.project_id),
-            compositions=repositories.Compositions(client_api=self._client_api, project=self._project)
+            compositions=repositories.Compositions(client_api=self._client_api, project=self._project),
+            models=repositories.Models(client_api=self._client_api, app=self)
         )
 
     @property
@@ -80,6 +82,11 @@ class App(entities.BaseEntity):
         assert isinstance(self._repositories.apps, repositories.Apps)
         return self._repositories.apps
 
+    @property
+    def models(self):
+        assert isinstance(self._repositories.models, repositories.Models)
+        return self._repositories.models
+
     @property
     def compositions(self):
         assert isinstance(self._repositories.compositions, repositories.Compositions)
@@ -141,15 +148,15 @@ class App(entities.BaseEntity):
         :return:
         """
         try:
-            package = App.from_json(_json=_json,
-                                    client_api=client_api,
-                                    project=project,
-                                    is_fetched=is_fetched)
+            app = App.from_json(_json=_json,
+                                client_api=client_api,
+                                project=project,
+                                is_fetched=is_fetched)
             status = True
         except Exception:
-            package = traceback.format_exc()
+            app = traceback.format_exc()
             status = False
-        return status, package
+        return status, app
 
     def to_json(self):
         _json = {}
@@ -189,6 +196,8 @@ class App(entities.BaseEntity):
             _json['settings'] = self.settings
         if self.integrations is not None:
             _json['integrations'] = self.integrations
+        if self.dpk is not None:
+            _json['dpk'] = self.dpk.to_json()
 
         return _json
 
@@ -214,7 +223,8 @@ class App(entities.BaseEntity):
             metadata=_json.get('metadata', None),
             status=_json.get('status', None),
             settings=_json.get('settings', {}),
-            integrations=_json.get('integrations', None)
+            integrations=_json.get('integrations', None),
+            dpk=entities.Dpk.from_json(_json=_json.get('dpk', {}), client_api=client_api, project=project, is_fetched=is_fetched)
         )
         app.is_fetched = is_fetched
         return app

dtlpy-1.119.5/dtlpy/entities/compute.py

@@ -0,0 +1,241 @@
+import traceback
+from enum import Enum
+from typing import List, Dict
+from ..services.api_client import ApiClient
+from .. import repositories, entities
+
+
+class ClusterProvider(str, Enum):
+    GCP = 'gcp'
+    AWS = 'aws'
+    AZURE = 'azure'
+    HPC = 'hpc'
+    LOCAL = 'local'
+    RANCHER_K3S = 'rancher-k3s'
+    RANCHER_RKE = 'rancher-rke'
+    OPENSHIFT = 'openshift'
+
+
+class ComputeType(str, Enum):
+    KUBERNETES = "kubernetes"
+
+
+class ComputeStatus(str, Enum):
+    READY = "ready"
+    INITIALIZING = "initializing"
+    PAUSE = "pause"
+    FAILED = "failed"
+    VALIDATING = "validating"
+
+
+class ComputeConsumptionMethod(str, Enum):
+    MQ = "MQ",
+    API = "API"
+
+
+class ComputeSettings(entities.DlEntity):
+    default_namespace: str = entities.DlProperty(location=['defaultNamespace'], _type=str)
+    consumption_method: str = entities.DlProperty(location=['consumptionMethod'], _type=str)
+
+    def to_json(self) -> dict:
+        return self._dict.copy()
+
+    @classmethod
+    def from_json(cls, _json):
+        return cls(_dict=_json)
+
+
+class Toleration(entities.DlEntity):
+    effect: str = entities.DlProperty(location=['effect'], _type=str)
+    key: str = entities.DlProperty(location=['key'], _type=str)
+    operator: str = entities.DlProperty(location=['operator'], _type=str)
+    value: str = entities.DlProperty(location=['value'], _type=str)
+
+    def to_json(self) -> dict:
+        return self._dict.copy()
+
+    @classmethod
+    def from_json(cls, _json):
+        return cls(_dict=_json)
+
+
+class DeploymentResource(entities.DlEntity):
+    gpu: int = entities.DlProperty(location=['gpu'], _type=int)
+    cpu: int = entities.DlProperty(location=['cpu'], _type=int)
+    memory: str = entities.DlProperty(location=['memory'], _type=str)
+
+    def to_json(self) -> dict:
+        return self._dict.copy()
+
+    @classmethod
+    def from_json(cls, _json):
+        return cls(_dict=_json)
+
+
+class DeploymentResources(entities.DlEntity):
+    request: DeploymentResource = entities.DlProperty(location=['request'], _kls='DeploymentResource')
+    limit: DeploymentResource = entities.DlProperty(location=['limit'], _kls='DeploymentResource')
+
+    def to_json(self) -> dict:
+        return self._dict.copy()
+
+    @classmethod
+    def from_json(cls, _json):
+        return cls(_dict=_json)
+
+
+class NodePool(entities.DlEntity):
+    name: str = entities.DlProperty(location=['name'], _type=str)
+    is_dl_type_default: bool = entities.DlProperty(location=['isDlTypeDefault'], _type=bool)
+    is_monitoring_configuration: bool = entities.DlProperty(location=['isMonitoringConfiguration'], _type=bool)
+    dl_types: List[str] = entities.DlProperty(location=['dlTypes'], _type=list)
+    tolerations: List[Toleration] = entities.DlProperty(location=['tolerations'], _kls='Toleration', default=[])
+    description: str = entities.DlProperty(location=['description'], _type=str, default='')
+    node_selector: str = entities.DlProperty(location=['nodeSelector'], _type=str, default='')
+    preemptible: bool = entities.DlProperty(location=['preemptible'], _type=bool, default=False)
+    deployment_resources: DeploymentResources = entities.DlProperty(location=['deploymentResources'], _kls='DeploymentResources')
+
+    def to_json(self) -> dict:
+        return self._dict.copy()
+
+    @classmethod
+    def from_json(cls, _json):
+        return cls(_dict=_json)
+
+
+class AuthenticationIntegration(entities.DlEntity):
+    id: str = entities.DlProperty(location=['id'], _type=str)
+    type: str = entities.DlProperty(location=['type'], _type=str)
+
+    def to_json(self) -> dict:
+        return self._dict.copy()
+
+    @classmethod
+    def from_json(cls, _json):
+        return cls(_dict=_json)
+
+
+class Authentication(entities.DlEntity):
+    integration: AuthenticationIntegration = entities.DlProperty(location=['integration'], _kls='AuthenticationIntegration')
+
+    def to_json(self) -> dict:
+        return self._dict.copy()
+
+    @classmethod
+    def from_json(cls, _json):
+        return cls(_dict=_json)
+
+
+class ComputeCluster(entities.DlEntity):
+    name: str = entities.DlProperty(location=['name'], _type=str)
+    endpoint: str = entities.DlProperty(location=['endpoint'], _type=str)
+    kubernetes_version: str = entities.DlProperty(location=['kubernetesVersion'], _type=str)
+    provider: str = entities.DlProperty(location=['provider'], _type=str)
+    node_pools: List[NodePool] = entities.DlProperty(location=['nodePools'], _kls='NodePool', default=[])
+    metadata: Dict = entities.DlProperty(location=['metadata'], _type=dict, default={})
+    authentication: Authentication = entities.DlProperty(location=['authentication'], _kls='Authentication')
+    plugins: dict = entities.DlProperty(location=['plugins'], _type=dict)
+    deployment_configuration: Dict = entities.DlProperty(location=['deploymentConfiguration'], _type=dict, default={})
+
+    def to_json(self) -> dict:
+        return self._dict.copy()
+
+    @classmethod
+    def from_json(cls, _json):
+        return cls(_dict=_json)
+
+    @classmethod
+    def from_setup_json(cls, devops_output, integration):
+        _json = {
+            'name': devops_output['config']['name'],
+            'endpoint': devops_output['config']['endpoint'],
+            'kubernetesVersion': devops_output['config']['kubernetesVersion'],
+            'provider': devops_output['config']['provider'],
+            'nodePools': devops_output['config']['nodePools'],
+            'metadata': {},
+            'authentication': {'integration': {'id': integration.id, 'type': integration.type}},
+            'plugins': devops_output['config'].get('plugins'),
+            'deploymentConfiguration': devops_output['config'].get('deploymentConfiguration', {})
+        }
+        return cls(_dict=_json)
+
+
+class ComputeContext(entities.DlEntity):
+    labels: List[str] = entities.DlProperty(location=['labels'], _type=list, default=[])
+    org: str = entities.DlProperty(location=['org'], _type=str)
+    project: str = entities.DlProperty(location=['project'], _type=str)
+
+    def to_json(self) -> dict:
+        return self._dict.copy()
+
+    @classmethod
+    def from_json(cls, _json):
+        return cls(_dict=_json)
+
+
+class Compute(entities.DlEntity):
+    id: str = entities.DlProperty(location=['id'], _type=str)
+    name: str = entities.DlProperty(location=['name'], _type=str)
+    context: ComputeContext = entities.DlProperty(location=['context'], _kls='ComputeContext')
+    shared_contexts: List[ComputeContext] = entities.DlProperty(location=['sharedContexts'], _kls='ComputeContext', default=[])
+    global_: bool = entities.DlProperty(location=['global'], _type=bool)
+    status: str = entities.DlProperty(location=['status'], _type=str, default=ComputeStatus.INITIALIZING.value)
+    type: str = entities.DlProperty(location=['type'], _type=str, default=ComputeType.KUBERNETES.value)
+    features: Dict = entities.DlProperty(location=['features'], _type=dict, default={})
+    metadata: Dict = entities.DlProperty(location=['metadata'], _type=dict, default={})
+    settings: ComputeSettings = entities.DlProperty(location=['settings'], _kls='ComputeSettings')
+    url: str = entities.DlProperty(location=['url'], _type=str)
+
+    def __init__(self, _dict=None, client_api: ApiClient = None, **kwargs):
+        super().__init__(_dict=_dict, **kwargs)
+        self._client_api = client_api
+        self._computes = None
+        self._serviceDrivers = None
+
+    @property
+    def computes(self):
+        if self._computes is None:
+            self._computes = repositories.Computes(client_api=self._client_api)
+        return self._computes
+
+    def delete(self, skip_destroy: bool = False):
+        return self.computes.delete(compute_id=self.id, skip_destroy=skip_destroy)
+
+    def update(self):
+        return self.computes.update(compute=self)
+
+    @staticmethod
+    def _protected_from_json(_json: dict, client_api: ApiClient):
+        """
+        Same as from_json but with try-except to catch if error
+
+        :param _json: platform json
+        :param client_api: ApiClient entity
+        :return:
+        """
+        try:
+            compute = Compute.from_json(_json=_json,
+                                        client_api=client_api)
+            status = True
+        except Exception:
+            compute = traceback.format_exc()
+            status = False
+        return status, compute
+
+    @classmethod
+    def from_json(cls, _json, client_api: ApiClient):
+        return cls(_dict=_json, client_api=client_api)
+
+    def to_json(self) -> dict:
+        return self._dict.copy()
+
+
+class KubernetesCompute(Compute):
+    cluster: ComputeCluster = entities.DlProperty(location=['cluster'], _kls='ComputeCluster')
+
+    @classmethod
+    def from_json(cls, _json, client_api: ApiClient):
+        return cls(_dict=_json, client_api=client_api)
+
+    def to_json(self) -> dict:
+        return self._dict.copy()

{dtlpy-1.118.15 → dtlpy-1.119.5}/dtlpy/entities/dpk.py

@@ -1,10 +1,15 @@
 from collections import namedtuple
-from typing import List, Union
+from typing import List, Union, Tuple
 import traceback
 import enum
+import tempfile
+import inspect
+import os
+import typing
 
-from .. import entities, repositories, exceptions
+from .. import entities, repositories, exceptions, assets
 from ..services.api_client import ApiClient
+from . import package_defaults
 
 
 class SlotType(str, enum.Enum):
@@ -29,6 +34,18 @@ DEFAULT_STOPS = {SlotType.ITEM_VIEWER: {"type": "itemViewer",
                                             }
                  }
 
+DEFAULT_RUNTIME = {
+    "podType": "regular-xs",
+    "concurrency": 1,
+    "runnerImage": "docker.io/dataloopai/dtlpy-agent:cpu.py3.10.opencv",
+    "autoscaler": {
+        "type": "rabbitmq",
+        "minReplicas": 0,
+        "maxReplicas": 2,
+        "queueLength": 100
+    }
+}
+
 
 class Slot(entities.DlEntity):
     type: str = entities.DlProperty(location=['type'], _type=str)
@@ -468,3 +485,299 @@ class Dpk(entities.DlEntity):
         )
 
         return res
+
+    @staticmethod
+    def _parse_function_io(func) -> Tuple[List[dict], List[dict]]:
+        """
+        Parse function inputs and outputs from function signature and type hints.
+        
+        This helper function extracts:
+        - Inputs: from function parameters (inspect.signature)
+        - Outputs: from return type hints (typing.get_type_hints)
+        
+        :param func: The function to parse
+        :return: Tuple of (inputs, outputs) where each is a list of dicts with 'name' and 'type' keys
+        :rtype: Tuple[List[dict], List[dict]]
+        """
+        # ============================================
+        # PARSE INPUTS FROM FUNCTION SIGNATURE
+        # ============================================
+        # Get all parameter names from the function signature
+        params = list(inspect.signature(func).parameters)
+        inputs = []
+        
+        # Create a mapping of known input type names (lowercase) to PackageInputType values
+        # This allows matching parameter names like 'item', 'items', 'annotation', etc. to their types
+        inputs_types = {i.name.lower(): i.value for i in list(entities.PackageInputType)}
+        
+        # Process each parameter
+        for arg in params:
+            # Check if the parameter name matches a known PackageInputType (case-insensitive)
+            # For example: 'item' -> PackageInputType.ITEM, 'items' -> PackageInputType.ITEMS
+            if arg in inputs_types:
+                inpt_type = inputs_types[arg]
+            else:
+                # If parameter name doesn't match a known type, default to JSON
+                inpt_type = entities.PackageInputType.JSON
+            
+            # Create input dict with name and type
+            inputs.append({
+                'name': arg,
+                'type': inpt_type
+            })
+        
+        # ============================================
+        # PARSE OUTPUTS FROM TYPE HINTS
+        # ============================================
+        outputs = []
+        try:
+            # Get return type hint from function annotations
+            # Returns None if no return type is specified
+            hint_outputs = typing.get_type_hints(func).get('return', None)
+            
+            if hint_outputs is not None:
+                # ============================================
+                # STEP 1: Extract output types from type hint
+                # ============================================
+                # Handle different return type patterns:
+                # - Tuple[Type1, Type2, ...] -> multiple outputs
+                # - List[Type] -> single output that's a list
+                # - Type -> single output
+                output_types = []
+                
+                if hasattr(hint_outputs, '__origin__') and hint_outputs.__origin__ is tuple:
+                    # Multiple returns: Tuple[Type1, Type2, ...]
+                    # Extract all types from the tuple
+                    output_types = list(hint_outputs.__args__) if hasattr(hint_outputs, '__args__') else []
+                elif hasattr(hint_outputs, '__args__'):
+                    # Single generic type (e.g., List[Item], Dict[str, int])
+                    # Extract the inner type from the generic
+                    output_types = [hint_outputs.__args__[0]]
+                else:
+                    # Single simple type (e.g., Item, str, int)
+                    output_types = [hint_outputs]
+                
+                # ============================================
+                # STEP 2: Process each output type
+                # ============================================
+                for idx, output_type in enumerate(output_types):
+                    io_type = output_type
+                    is_list = False
+                    
+                    # ============================================
+                    # STEP 2a: Check if it's a List type
+                    # ============================================
+                    # Handle List[Type] patterns (e.g., List[Item], List[Annotation])
+                    if hasattr(io_type, '__origin__'):
+                        origin = io_type.__origin__
+                        origin_str = str(origin)
+                        
+                        # Check if origin is list or typing.List
+                        # This handles both 'list' and 'typing.List' cases
+                        if origin is list or origin_str.startswith('typing.List') or origin_str.startswith('<class \'list\'>'):
+                            is_list = True
+                            # Extract inner type from List[InnerType]
+                            # Example: List[Item] -> extract 'Item'
+                            if hasattr(io_type, '__args__') and len(io_type.__args__) > 0:
+                                io_type = io_type.__args__[0]
+                    
+                    # ============================================
+                    # STEP 2b: Handle special type cases
+                    # ============================================
+                    # Handle Enum types - convert to string name
+                    if isinstance(io_type, enum.Enum):
+                        io_type = io_type.name
+                    
+                    # Handle nested generic types (only if not already extracted from List)
+                    # Example: Optional[List[Item]] -> extract List[Item] first, then Item
+                    if not is_list and hasattr(io_type, '__args__'):
+                        io_type = io_type.__args__[0]
+                    
+                    # ============================================
+                    # STEP 2c: Extract type name as string
+                    # ============================================
+                    # Convert type to string representation
+                    if isinstance(io_type, type):
+                        # Direct type object (e.g., Item class) -> get class name
+                        type_str = io_type.__name__
+                    else:
+                        # Type hint object -> convert to string
+                        type_str = str(io_type)
+                        # Extract class name from fully qualified name
+                        # Example: "dtlpy.entities.item.Item" -> "Item"
+                        if '.' in type_str:
+                            type_str = type_str.rsplit('.', maxsplit=1)[-1]
+                    
+                    # ============================================
+                    # STEP 2d: Map type string to PackageInputType
+                    # ============================================
+                    # Map Python type names to PackageInputType enum values
+                    # Special case: 'str' maps to STRING
+                    if type_str == 'str':
+                        mapped_type = entities.PackageInputType.STRING
+                    else:
+                        mapped_type = None
+                        
+                        # If it's a list, try to find the array version first
+                        # Example: Item -> ITEMS, Annotation -> ANNOTATIONS
+                        if is_list:
+                            array_type_name = f"{type_str.upper()}S"  # Item -> ITEMS
+                            for pkg_input_type in entities.PackageInputType:
+                                if pkg_input_type.name.upper() == array_type_name:
+                                    mapped_type = pkg_input_type
+                                    break
+                        
+                        # If not found or not a list, try the regular type
+                        # Example: Item -> ITEM, Annotation -> ANNOTATION
+                        if mapped_type is None:
+                            for pkg_input_type in entities.PackageInputType:
+                                if pkg_input_type.name.upper() == type_str.upper():
+                                    mapped_type = pkg_input_type
+                                    break
+                        
+                        # Default to JSON if no match found
+                        # This handles unknown types gracefully
+                        if mapped_type is None:
+                            mapped_type = entities.PackageInputType.JSON
+                    
+                    # ============================================
+                    # STEP 2e: Create output dict
+                    # ============================================
+                    # Name outputs: 'output' for single, 'output_0', 'output_1', etc. for multiple
+                    output_name = 'output' if len(output_types) == 1 else f'output_{idx}'
+                    outputs.append({
+                        'name': output_name,
+                        'type': mapped_type
+                    })
+        except Exception:
+            # If type hints extraction fails (e.g., no type hints, invalid syntax),
+            # leave outputs empty - this is not a fatal error
+            pass
+        
+        return inputs, outputs
+
+    @classmethod
+    def from_function(
+        cls,
+        func,
+        name: str = None,
+        version: str = '1.0.0',
+        description: str = None,
+        project: 'entities.Project' = None,
+        client_api: ApiClient = None,
+        scope: str = 'project',
+        runtime: dict = None,
+        execution_timeout: int = 3600,
+    ) -> 'entities.Dpk':
+        """
+        Build a DPK from a Python function (codebase + components). Does not publish or install.
+        Runtime is held on the service component (modules have no runtime). Use the runtime param to override defaults.
+
+        :param Callable func: Function to deploy
+        :param str name: DPK name (default: function __name__)
+        :param str version: DPK version (default: '1.0.0')
+        :param str description: DPK description
+        :param entities.Project project: Project (required)
+        :param ApiClient client_api: API client (required)
+        :param str scope: 'project' or 'organization'
+        :param dict runtime: podType, concurrency, runnerImage, autoscaler; default: DEFAULT_RUNTIME
+        :param int execution_timeout: Service timeout in seconds (default: 3600)
+        :return: Built DPK (with codebase); use Service.from_function to publish, install, and get the Service
+        :rtype: dtlpy.entities.Dpk
+        """
+        if client_api is None:
+            raise ValueError('client_api is required')
+        if project is None:
+            raise ValueError('project is required for codebase packing')
+
+        # Get function name if name not provided
+        if name is None:
+            name = func.__name__
+
+        # Resolve runtime: use copy to avoid mutating caller's dict or DEFAULT_RUNTIME
+        if runtime is None:
+            runtime = dict(DEFAULT_RUNTIME)
+        else:
+            runtime = dict(runtime)
+
+        # Create temporary directory for the function code
+        dpk_dir = tempfile.mkdtemp()
+
+        # Create main.py file with the function
+        main_file = os.path.join(dpk_dir, package_defaults.DEFAULT_PACKAGE_ENTRY_POINT)
+
+        # Read the partial main template (ServiceRunner class structure)
+        with open(assets.paths.PARTIAL_MAIN_FILEPATH, 'r') as f:
+            main_template = f.read()
+
+        # Extract function source code and adjust indentation
+        lines = inspect.getsourcelines(func)
+        tabs_diff = lines[0][0].count('    ') - 1
+        for line_index in range(len(lines[0])):
+            line_tabs = lines[0][line_index].count('    ') - tabs_diff
+            lines[0][line_index] = ('    ' * line_tabs) + lines[0][line_index].strip() + '\n'
+
+        method_func_string = "".join(lines[0])
+
+        # Write main.py with ServiceRunner class and function
+        with open(main_file, 'w') as f:
+            f.write(f'{main_template}\n    @staticmethod\n{method_func_string}')
+
+        # Parse function inputs and outputs from signature and type hints
+        inputs, outputs = cls._parse_function_io(func)
+
+        # Build module dict (no runtime on module/functions)
+        function_dict = {
+            'name': func.__name__,
+            'input': inputs,
+            'output': outputs,
+        }
+        module_dict = {
+            'name': package_defaults.DEFAULT_PACKAGE_MODULE_NAME,
+            'entryPoint': package_defaults.DEFAULT_PACKAGE_ENTRY_POINT,
+            'className': package_defaults.DEFAULT_PACKAGE_CLASS_NAME,
+            'functions': [function_dict],
+        }
+
+        # Service component (smart-search style): holds runtime, references module
+        service_name = f"{name}-service".replace("_", "-")
+        service_entry = {
+            "name": service_name,
+            "moduleName": package_defaults.DEFAULT_PACKAGE_MODULE_NAME,
+            "packageRevision": "latest",
+            "runtime": runtime,
+            "executionTimeout": execution_timeout,
+        }
+
+        components_dict = {
+            "modules": [module_dict],
+            "services": [service_entry],
+        }
+
+        # Create DPK entity
+        dpk_json = {
+            'name': name,
+            'version': version,
+            'display_name': name,
+            'description': description or f'DPK created from function {func.__name__}',
+            'scope': scope,
+            'components': components_dict,
+            'context': {
+                'project': project.id
+            }
+        }
+
+        dpk = cls.from_json(
+            _json=dpk_json,
+            client_api=client_api,
+            project=project
+        )
+
+        # Pack the temporary directory as codebase
+        dpk.codebase = project.codebases.pack(
+            directory=dpk_dir,
+            name=name,
+            extension='dpk'
+        )
+
+        return dpk