aipmodel 0.2.65__tar.gz → 0.2.67__tar.gz

{aipmodel-0.2.65 → aipmodel-0.2.67}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aipmodel
-Version: 0.2.65
+Version: 0.2.67
 Summary: SDK for model registration, versioning, and storage
 Home-page: https://github.com/AIP-MLOPS/model-registry
 Author: AIP MLOPS Team

{aipmodel-0.2.65 → aipmodel-0.2.67}/README.md

@@ -51,6 +51,12 @@ S3_MAX_CONCURRENCY="10"
 S3_UPLOAD_WORKERS="4"
 S3_DOWNLOAD_WORKERS="4"
 
+# Auto-retry for transient connection drops during large transfers
+S3_MAX_RETRIES="5"               # per-file retries before giving up
+S3_RETRY_BACKOFF_SECONDS="2"     # base backoff, grows exponentially per attempt
+S3_RETRY_MAX_DELAY_SECONDS="30"  # cap on a single backoff sleep
+S3_BOTO_MAX_ATTEMPTS="10"        # botocore request-level retry attempts (adaptive)
+
 ```
 
 > **Note:** `CLEARML_WEB_HOST` is new. If not provided, the SDK will default to the API host, which might result in incorrect clickable links in the output.
@@ -68,6 +74,50 @@ S3_DOWNLOAD_WORKERS="4"
 
 Nothing is ever logged to a file or hidden — all output goes to stdout via `print()`, so it's visible wherever the process runs.
 
+## Resilient Transfers (Auto-Retry)
+
+Large multi-GB uploads/downloads over Ceph/RGW occasionally hit a **transient**
+connection drop mid-stream (`IncompleteRead` / `ProtocolError` / connection
+reset). The SDK now **retries these automatically**, so `add_model`,
+`get_model` and `download` complete without manual intervention:
+
+- Each **individual file** transfer is retried on a transient error with
+  exponential backoff (`S3_RETRY_BACKOFF_SECONDS`, capped by
+  `S3_RETRY_MAX_DELAY_SECONDS`), up to `S3_MAX_RETRIES` times. Downloads reset
+  the partial file before each retry so no corrupt bytes remain.
+- The underlying boto3 client also uses botocore's **adaptive** request-level
+  retries (`S3_BOTO_MAX_ATTEMPTS`).
+- **Non-transient** errors (bad input, auth, not-found) are **not** retried —
+  they fail immediately as before.
+- Fully backward compatible: method signatures, return values and the final
+  `ValueError` raised on ultimate failure are unchanged. All knobs are
+  `.env`-overridable with sane defaults.
+
+## Concurrent Adds
+
+`add_model` is **concurrency-safe** — you can run many adds at the same time:
+
+- Adds with **different model names** run **fully in parallel** (own ClearML id,
+  own S3 prefix — no interference).
+- Adds with the **same `(project, model_name)`** are **serialized** by a
+  process-wide lock. Otherwise two simultaneous same-name adds would race on the
+  existence check + auto version-number computation and could create a
+  **duplicate model** or **overwrite a version**. With the lock the second add
+  waits for the first, then correctly continues to the next version
+  (`_v2` → `_v3`, …) — no duplicates, no lost versions.
+- Uploads are **staged**: files go to a temp S3 path first and are only moved to
+  the final version path once the whole upload succeeds; any failure rolls back
+  (deletes the new model if it was new + the temp folder) and re-raises, so a
+  crashed/aborted add never leaves a half-written version.
+- **Duplicate `(model_name, version)` is rejected**: re-adding an existing
+  explicit version raises `ValueError` instead of silently overwriting it (omit
+  `version` to auto-generate the next unique one).
+- **Orphaned temp folders** left behind by a hard-killed upload are swept
+  automatically on the next add to that model.
+- The lock is **per-process**. One server process (e.g. a single `uvicorn`
+  worker) is fully protected; for multiple workers/replicas add a distributed
+  lock (ClearML/Redis).
+
 ## Access Control (ACL)
 
 ACL is **off by default** and fully optional. Toggle it with one env var:

{aipmodel-0.2.65 → aipmodel-0.2.67}/aipmodel/__init__.py

@@ -6,5 +6,5 @@ check_latest_version("aipmodel")
 
 __all__ = ["MLOpsManager", "CephS3Manager"]
 
-__version__ = "0.2.65"
+__version__ = "0.2.67"
 __description__ = "SDK for model registration, versioning, and storage"

{aipmodel-0.2.65 → aipmodel-0.2.67}/aipmodel/model_registry.py

@@ -10,6 +10,7 @@ from huggingface_hub import snapshot_download
 from typing import Optional, List, Dict, Any
 import json
 import logging
+import threading
 import time
 import re
 from tqdm import tqdm
@@ -22,6 +23,25 @@ load_dotenv()
 
 logger = logging.getLogger(__name__)
 
+# Process-wide locks that serialize concurrent add_model calls targeting the SAME
+# (project, model_name). Two simultaneous adds of the same name would otherwise
+# race on the existence check + version-number computation and could create a
+# duplicate model or clobber a version. Different names lock on different keys and
+# still run fully in parallel. NOTE: per-process — with multiple API workers /
+# replicas a distributed lock would be required for the same guarantee.
+_ADD_MODEL_LOCKS = {}
+_ADD_MODEL_LOCKS_GUARD = threading.Lock()
+
+
+def _get_add_model_lock(project_id, model_name):
+    key = (project_id, model_name)
+    with _ADD_MODEL_LOCKS_GUARD:
+        lock = _ADD_MODEL_LOCKS.get(key)
+        if lock is None:
+            lock = threading.Lock()
+            _ADD_MODEL_LOCKS[key] = lock
+        return lock
+
 class ProjectsAPI:
     def __init__(self, post, verbose):
         self.verbose = verbose
@@ -667,9 +687,59 @@ class MLOpsManager:
             if os.path.exists(tmp_dir):
                 shutil.rmtree(tmp_dir)
 
+    def _cleanup_orphan_temp_folders(self, manager, model_id, keep_prefix=None):
+        """
+        Best-effort removal of leftover `*_tmp_*` staging folders under a model —
+        orphans from a previously interrupted/crashed upload (where rollback
+        never ran). Safe because add_model holds a per-(project, model_name)
+        lock, so no other add for this model is running concurrently. Never
+        raises: cleanup must never break an otherwise-valid add.
+        """
+        try:
+            base = f"_models/{model_id}/"
+            keys = manager.list_folder_contents(base)
+            temp_prefixes = set()
+            for key in keys:
+                rel = key[len(base):] if key.startswith(base) else key
+                top = rel.split("/", 1)[0]
+                prefix = f"{base}{top}/"
+                if "_tmp_" in top and prefix != keep_prefix:
+                    temp_prefixes.add(prefix)
+            for prefix in temp_prefixes:
+                try:
+                    manager.delete_folder(prefix)
+                    if self.verbose:
+                        print(f"[SDK_INFO] Cleaned orphan temp folder: {prefix}")
+                except Exception:
+                    pass
+        except Exception:
+            pass
+
     def add_model(self, source_type, model_name=None, version=None, source_path=None, code_path=None, category=None, tags=None,
                   public=False, external_ceph_endpoint_url=None, external_ceph_bucket_name=None, external_ceph_access_key=None, external_ceph_secret_key=None,
                   model_folder_name=None):
+        """
+        Register/add a model version. Concurrency-safe: adds that target the SAME
+        (project, model_name) are serialized by a process-wide lock so simultaneous
+        same-name adds cannot create a duplicate model or collide on the auto
+        version number; adds with different names run fully in parallel. All logic
+        lives in _add_model_impl — signature, behavior and return value are
+        unchanged (this wrapper only adds the lock).
+        """
+        with _get_add_model_lock(self.project_id, model_name):
+            return self._add_model_impl(
+                source_type, model_name=model_name, version=version, source_path=source_path,
+                code_path=code_path, category=category, tags=tags, public=public,
+                external_ceph_endpoint_url=external_ceph_endpoint_url,
+                external_ceph_bucket_name=external_ceph_bucket_name,
+                external_ceph_access_key=external_ceph_access_key,
+                external_ceph_secret_key=external_ceph_secret_key,
+                model_folder_name=model_folder_name,
+            )
+
+    def _add_model_impl(self, source_type, model_name=None, version=None, source_path=None, code_path=None, category=None, tags=None,
+                  public=False, external_ceph_endpoint_url=None, external_ceph_bucket_name=None, external_ceph_access_key=None, external_ceph_secret_key=None,
+                  model_folder_name=None):
 
         if self.verbose:
             print(f"[SDK_INFO] Adding model: {model_name} (Source: {source_type}, Public: {public})")
@@ -775,6 +845,17 @@ class MLOpsManager:
                             max_version_number = vn
                     except Exception:
                         pass
+
+            # Reject a duplicate (model_name, version): if the caller passed an
+            # explicit version that already exists, fail instead of silently
+            # overwriting it. Omitting version (auto-generation) is always unique.
+            if version and version in versions_map:
+                error_message = (
+                    f"Version '{version}' already exists for model '{model_name}'. "
+                    f"Choose a different version name, or omit 'version' to auto-generate the next one."
+                )
+                print(f"[SDK_FAIL] {error_message}")
+                raise ValueError(error_message)
         else:
             if self.verbose:
                 print("[SDK_INFO] Creating new model entry...")
@@ -803,6 +884,11 @@ class MLOpsManager:
             else:
                 model_folder_name = "unknown"
 
+        # Sweep orphaned temp folders from any previously interrupted upload
+        # (safe under the per-name lock). New models have none.
+        if not is_new_model:
+            self._cleanup_orphan_temp_folders(current_manager, model_id)
+
         dest_version_path = f"_models/{model_id}/{version}/"
         temp_suffix = self.generate_random_string()
         dest_temp_path = f"_models/{model_id}/{version}_tmp_{temp_suffix}/"

{aipmodel-0.2.65 → aipmodel-0.2.67}/aipmodel.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aipmodel
-Version: 0.2.65
+Version: 0.2.67
 Summary: SDK for model registration, versioning, and storage
 Home-page: https://github.com/AIP-MLOPS/model-registry
 Author: AIP MLOPS Team

{aipmodel-0.2.65 → aipmodel-0.2.67}/setup.py

@@ -31,7 +31,7 @@ version, description = read_meta()
 
 setup(
     name="aipmodel",
-    version="0.2.65",
+    version="0.2.67",
     description=description,
     author="AIP MLOPS Team",
     author_email="mohmmadweb@gmail.com",