aipmodel 0.2.65__tar.gz → 0.2.66__tar.gz

{aipmodel-0.2.65 → aipmodel-0.2.66}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aipmodel
-Version: 0.2.65
+Version: 0.2.66
 Summary: SDK for model registration, versioning, and storage
 Home-page: https://github.com/AIP-MLOPS/model-registry
 Author: AIP MLOPS Team

{aipmodel-0.2.65 → aipmodel-0.2.66}/README.md

@@ -51,6 +51,12 @@ S3_MAX_CONCURRENCY="10"
 S3_UPLOAD_WORKERS="4"
 S3_DOWNLOAD_WORKERS="4"
 
+# Auto-retry for transient connection drops during large transfers
+S3_MAX_RETRIES="5"               # per-file retries before giving up
+S3_RETRY_BACKOFF_SECONDS="2"     # base backoff, grows exponentially per attempt
+S3_RETRY_MAX_DELAY_SECONDS="30"  # cap on a single backoff sleep
+S3_BOTO_MAX_ATTEMPTS="10"        # botocore request-level retry attempts (adaptive)
+
 ```
 
 > **Note:** `CLEARML_WEB_HOST` is new. If not provided, the SDK will default to the API host, which might result in incorrect clickable links in the output.
@@ -68,6 +74,45 @@ S3_DOWNLOAD_WORKERS="4"
 
 Nothing is ever logged to a file or hidden — all output goes to stdout via `print()`, so it's visible wherever the process runs.
 
+## Resilient Transfers (Auto-Retry)
+
+Large multi-GB uploads/downloads over Ceph/RGW occasionally hit a **transient**
+connection drop mid-stream (`IncompleteRead` / `ProtocolError` / connection
+reset). The SDK now **retries these automatically**, so `add_model`,
+`get_model` and `download` complete without manual intervention:
+
+- Each **individual file** transfer is retried on a transient error with
+  exponential backoff (`S3_RETRY_BACKOFF_SECONDS`, capped by
+  `S3_RETRY_MAX_DELAY_SECONDS`), up to `S3_MAX_RETRIES` times. Downloads reset
+  the partial file before each retry so no corrupt bytes remain.
+- The underlying boto3 client also uses botocore's **adaptive** request-level
+  retries (`S3_BOTO_MAX_ATTEMPTS`).
+- **Non-transient** errors (bad input, auth, not-found) are **not** retried —
+  they fail immediately as before.
+- Fully backward compatible: method signatures, return values and the final
+  `ValueError` raised on ultimate failure are unchanged. All knobs are
+  `.env`-overridable with sane defaults.
+
+## Concurrent Adds
+
+`add_model` is **concurrency-safe** — you can run many adds at the same time:
+
+- Adds with **different model names** run **fully in parallel** (own ClearML id,
+  own S3 prefix — no interference).
+- Adds with the **same `(project, model_name)`** are **serialized** by a
+  process-wide lock. Otherwise two simultaneous same-name adds would race on the
+  existence check + auto version-number computation and could create a
+  **duplicate model** or **overwrite a version**. With the lock the second add
+  waits for the first, then correctly continues to the next version
+  (`_v2` → `_v3`, …) — no duplicates, no lost versions.
+- Uploads are **staged**: files go to a temp S3 path first and are only moved to
+  the final version path once the whole upload succeeds; any failure rolls back
+  (deletes the new model if it was new + the temp folder) and re-raises, so a
+  crashed/aborted add never leaves a half-written version.
+- The lock is **per-process**. One server process (e.g. a single `uvicorn`
+  worker) is fully protected; for multiple workers/replicas add a distributed
+  lock (ClearML/Redis).
+
 ## Access Control (ACL)
 
 ACL is **off by default** and fully optional. Toggle it with one env var:

{aipmodel-0.2.65 → aipmodel-0.2.66}/aipmodel/__init__.py

@@ -6,5 +6,5 @@ check_latest_version("aipmodel")
 
 __all__ = ["MLOpsManager", "CephS3Manager"]
 
-__version__ = "0.2.65"
+__version__ = "0.2.66"
 __description__ = "SDK for model registration, versioning, and storage"

{aipmodel-0.2.65 → aipmodel-0.2.66}/aipmodel/model_registry.py

@@ -10,6 +10,7 @@ from huggingface_hub import snapshot_download
 from typing import Optional, List, Dict, Any
 import json
 import logging
+import threading
 import time
 import re
 from tqdm import tqdm
@@ -22,6 +23,25 @@ load_dotenv()
 
 logger = logging.getLogger(__name__)
 
+# Process-wide locks that serialize concurrent add_model calls targeting the SAME
+# (project, model_name). Two simultaneous adds of the same name would otherwise
+# race on the existence check + version-number computation and could create a
+# duplicate model or clobber a version. Different names lock on different keys and
+# still run fully in parallel. NOTE: per-process — with multiple API workers /
+# replicas a distributed lock would be required for the same guarantee.
+_ADD_MODEL_LOCKS = {}
+_ADD_MODEL_LOCKS_GUARD = threading.Lock()
+
+
+def _get_add_model_lock(project_id, model_name):
+    key = (project_id, model_name)
+    with _ADD_MODEL_LOCKS_GUARD:
+        lock = _ADD_MODEL_LOCKS.get(key)
+        if lock is None:
+            lock = threading.Lock()
+            _ADD_MODEL_LOCKS[key] = lock
+        return lock
+
 class ProjectsAPI:
     def __init__(self, post, verbose):
         self.verbose = verbose
@@ -670,6 +690,28 @@ class MLOpsManager:
     def add_model(self, source_type, model_name=None, version=None, source_path=None, code_path=None, category=None, tags=None,
                   public=False, external_ceph_endpoint_url=None, external_ceph_bucket_name=None, external_ceph_access_key=None, external_ceph_secret_key=None,
                   model_folder_name=None):
+        """
+        Register/add a model version. Concurrency-safe: adds that target the SAME
+        (project, model_name) are serialized by a process-wide lock so simultaneous
+        same-name adds cannot create a duplicate model or collide on the auto
+        version number; adds with different names run fully in parallel. All logic
+        lives in _add_model_impl — signature, behavior and return value are
+        unchanged (this wrapper only adds the lock).
+        """
+        with _get_add_model_lock(self.project_id, model_name):
+            return self._add_model_impl(
+                source_type, model_name=model_name, version=version, source_path=source_path,
+                code_path=code_path, category=category, tags=tags, public=public,
+                external_ceph_endpoint_url=external_ceph_endpoint_url,
+                external_ceph_bucket_name=external_ceph_bucket_name,
+                external_ceph_access_key=external_ceph_access_key,
+                external_ceph_secret_key=external_ceph_secret_key,
+                model_folder_name=model_folder_name,
+            )
+
+    def _add_model_impl(self, source_type, model_name=None, version=None, source_path=None, code_path=None, category=None, tags=None,
+                  public=False, external_ceph_endpoint_url=None, external_ceph_bucket_name=None, external_ceph_access_key=None, external_ceph_secret_key=None,
+                  model_folder_name=None):
 
         if self.verbose:
             print(f"[SDK_INFO] Adding model: {model_name} (Source: {source_type}, Public: {public})")

{aipmodel-0.2.65 → aipmodel-0.2.66}/aipmodel.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aipmodel
-Version: 0.2.65
+Version: 0.2.66
 Summary: SDK for model registration, versioning, and storage
 Home-page: https://github.com/AIP-MLOPS/model-registry
 Author: AIP MLOPS Team

{aipmodel-0.2.65 → aipmodel-0.2.66}/setup.py

@@ -31,7 +31,7 @@ version, description = read_meta()
 
 setup(
     name="aipmodel",
-    version="0.2.65",
+    version="0.2.66",
     description=description,
     author="AIP MLOPS Team",
     author_email="mohmmadweb@gmail.com",