craft-ai-sdk 0.53.0__tar.gz → 0.54.0rc1__tar.gz

{craft_ai_sdk-0.53.0 → craft_ai_sdk-0.54.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: craft-ai-sdk
-Version: 0.53.0
+Version: 0.54.0rc1
 Summary: Craft AI MLOps platform SDK
 Home-page: https://www.craft.ai/
 License: Apache-2.0

{craft_ai_sdk-0.53.0 → craft_ai_sdk-0.54.0rc1}/craft_ai_sdk/__init__.py

@@ -14,4 +14,4 @@ from .io import (  # noqa: F401
 )
 
 
-__version__ = "0.53.0"
+__version__ = "0.54.0rc1"

{craft_ai_sdk-0.53.0 → craft_ai_sdk-0.54.0rc1}/craft_ai_sdk/core/pipelines.py

@@ -11,13 +11,12 @@ from ..utils import (
     handle_data_store_response,
     remove_keys_from_dict,
     _datetime_to_timestamp_in_ms,
+    _wait_create_until_ready,
 )
 from .steps import (
-    _add_inputs_outputs_in_message,
     _validate_create_step_parameters,
     _prepare_create_step_data,
     _prepare_create_step_files,
-    _wait_create_until_ready,
 )
 
 
@@ -43,6 +42,7 @@ def create_pipeline(
     container_config=None,
     inputs=None,
     outputs=None,
+    wait_for_completion=True,
     timeout_s=None,
     **kwargs,
 ):
@@ -114,8 +114,11 @@ def create_pipeline(
             :obj:`dict` with the name of the output as keys and the value of the
             output as values. Each output should be specified as an instance
             of :class:`Output` via this parameter `outputs`.
+        wait_for_completion (:obj:`bool`, optional): Whether to wait for the pipeline
+            to be created. Defaults to ``True``.
         timeout_s (:obj:`int`): Maximum time (in seconds) to wait for the pipeline to
-            be created. 3min (180s) by default, and at least 2min (120s).
+            be created. Set to None to wait indefinitely. Defaults to None.
+            Only applicable if ``wait_for_completion`` is ``True``.
 
     Returns:
         :obj:`dict`: Created pipeline represented as :obj:`dict` with the following
@@ -214,7 +217,6 @@ def create_pipeline(
 
     # Otherwise, the pipeline is created with a "hidden" step
     container_config = {} if container_config is None else container_config.copy()
-    timeout_s = 3 * 60 if timeout_s is None else timeout_s
     _validate_create_step_parameters(inputs, outputs, timeout_s)
 
     url = f"{sdk.base_environment_api_url}/pipelines"
@@ -235,31 +237,23 @@ def create_pipeline(
         "This may take a while...",
     )
 
-    start_time = sdk._get_time()
-    created_pipeline, response = sdk._post(
-        url, data=multipartify(data), files=files, get_response=True
-    )
+    sdk._post(url, data=multipartify(data), files=files, allow_redirects=False)
 
-    if response.status_code != 206:
-        message = _add_inputs_outputs_in_message(
-            f'Pipeline "{pipeline_name}" created',
-            created_pipeline.get("inputs", []),
-            created_pipeline.get("outputs", []),
-        )
-        log_action(sdk, message)
-        return remove_keys_from_dict(created_pipeline, {"is_archived", "pipeline_id"})
+    return get_pipeline(sdk, pipeline_name, wait_for_completion, timeout_s)
 
-    # The pipeline is still building. Keep checking its status until it is ready
-    return _wait_create_until_ready(
-        sdk, pipeline_name, get_pipeline, timeout_s, start_time, get_pipeline_logs
-    )
 
-
-def get_pipeline(sdk: BaseCraftAiSdk, pipeline_name):
+def get_pipeline(
+    sdk: BaseCraftAiSdk, pipeline_name, wait_for_completion=False, timeout_s=None
+):
     """Get a single pipeline if it exists.
 
     Args:
         pipeline_name (:obj:`str`): Name of the pipeline to get.
+        wait_for_completion (:obj:`bool`, optional): Whether to wait for the pipeline
+            to be created. Defaults to ``False``.
+        timeout_s (:obj:`int`): Maximum time (in seconds) to wait for the pipeline to
+            be created. Set to None to wait indefinitely. Defaults to None.
+            Only applicable if ``wait_for_completion`` is ``True``.
 
     Returns:
         None if the pipeline does not exist, otherwise pipeline information, with
@@ -322,8 +316,20 @@ def get_pipeline(sdk: BaseCraftAiSdk, pipeline_name):
             ``"git_repository"`` or ``"local"``.
     """
     try:
-        url = f"{sdk.base_environment_api_url}/pipelines/{pipeline_name}"
-        pipeline = sdk._get(url)
+        base_url = f"{sdk.base_environment_api_url}/pipelines/{pipeline_name}"
+        if wait_for_completion:
+            pipeline = _wait_create_until_ready(
+                sdk,
+                pipeline_name,
+                lambda sdk, _: sdk._get(
+                    f"{base_url}?wait_for_completion=true", allow_redirects=False
+                ),
+                timeout_s,
+                sdk._get_time(),
+                get_pipeline_logs,
+            )
+        else:
+            pipeline = sdk._get(base_url)
     except SdkException as error:
         if error.status_code == 404:
             return None

{craft_ai_sdk-0.53.0 → craft_ai_sdk-0.54.0rc1}/craft_ai_sdk/core/steps.py

@@ -1,4 +1,3 @@
-import time
 import tarfile
 import io
 import os
@@ -6,7 +5,6 @@ import requests
 
 from ..sdk import BaseCraftAiSdk
 from ..io import Input, Output
-from ..exceptions import SdkException
 from ..utils import (
     log_func_result,
     log_action,
@@ -15,8 +13,8 @@ from ..utils import (
     remove_none_values,
     handle_data_store_response,
     _datetime_to_timestamp_in_ms,
+    _wait_create_until_ready,
 )
-from ..constants import CREATION_REQUESTS_RETRY_INTERVAL
 
 
 def _compress_folder_to_memory(local_folder, include):
@@ -120,40 +118,6 @@ def _remove_id_from_step(step):
     return step
 
 
-def _wait_create_until_ready(sdk, name, get_func, timeout_s, start_time, get_log_func):
-    elapsed_time = sdk._get_time() - start_time
-    status = "creation_pending"
-    while status == "creation_pending" and (
-        timeout_s is None or elapsed_time < timeout_s
-    ):
-        time.sleep(CREATION_REQUESTS_RETRY_INTERVAL)
-        created_obj = get_func(sdk, name)
-        if created_obj is None:
-            raise SdkException(
-                f'The creation of "{name}" has failed. Please try creating the '
-                "resource again. If this error keeps happening, please contact the "
-                "support team.",
-                name="UnexpectedError",
-            )
-        status = created_obj.get("creation_info", {}).get("status", None)
-        elapsed_time = sdk._get_time() - start_time
-
-    if status == "creation_failed":
-        raise SdkException(
-            f'The creation of "{name}" has failed. You can check the logs with '
-            f'the "{get_log_func.__name__}" function.',
-            name="CreationFailed",
-        )
-    if status != "ready":
-        raise SdkException(
-            f'The creation of "{name}" was not ready in time. It is still being '
-            "created but this function stopped trying. Please check its status with "
-            f'"{get_func.__name__}".',
-            name="TimeoutException",
-        )
-    return created_obj
-
-
 @log_func_result("Steps creation")
 def create_step(
     sdk: BaseCraftAiSdk,

{craft_ai_sdk-0.53.0 → craft_ai_sdk-0.54.0rc1}/craft_ai_sdk/sdk.py

@@ -130,7 +130,7 @@ class CraftAiSdk(BaseCraftAiSdk):
         os.environ.get("CRAFT_AI__MULTIPART_PART_SIZE__B", str(38 * 256 * 1024))
     )
     _access_token_margin = timedelta(seconds=30)
-    _version = "0.53.0"  # Would be better to share it somewhere
+    _version = "0.54.0rc1"  # Would be better to share it somewhere
 
     def __init__(
         self,

{craft_ai_sdk-0.53.0 → craft_ai_sdk-0.54.0rc1}/craft_ai_sdk/utils.py

@@ -322,3 +322,29 @@ def multipartify(data, parent_key=None) -> dict:
             converted.append((current_key, formatter(value)))
 
     return dict(converted)
+
+
+def _wait_create_until_ready(sdk, name, get_func, timeout_s, start_time, get_log_func):
+    elapsed_time = sdk._get_time() - start_time
+    status = "creation_pending"
+    while status == "creation_pending" and (
+        timeout_s is None or elapsed_time < timeout_s
+    ):
+        created_obj = get_func(sdk, name)
+        status = created_obj.get("creation_info", {}).get("status", None)
+        elapsed_time = sdk._get_time() - start_time
+
+    if status == "creation_failed":
+        raise SdkException(
+            f'The creation of "{name}" has failed. You can check the logs with '
+            f'the "{get_log_func.__name__}" function.',
+            name="CreationFailed",
+        )
+    if status != "ready":
+        raise SdkException(
+            f'The creation of "{name}" was not ready in time. It is still being '
+            "created but this function stopped trying. Please check its status with "
+            f'"{get_func.__name__}".',
+            name="TimeoutException",
+        )
+    return created_obj

{craft_ai_sdk-0.53.0 → craft_ai_sdk-0.54.0rc1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "craft-ai-sdk"
-version = "0.53.0"
+version = "0.54.0rc1"
 description = "Craft AI MLOps platform SDK"
 license = "Apache-2.0"
 authors = ["Craft AI <contact@craft.ai>"]
@@ -34,7 +34,10 @@ StrEnum = "^0.4.9"
 ipython = "^8.0.1"
 ipykernel = "^6.9.1"
 black = "^22.1.0"
-flake8 = "^4.0.1"
+flake8 = [
+    { version = "^4.0.1", python = "3.8" },
+    { version = "^7.1.1", python = ">=3.9,<4.0" },
+]
 pylint = "^2.12.2"
 python-dotenv = "^1.0.0"
 pytest = "^7.0.1"