aipmodel 0.2.76__tar.gz → 0.2.77__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {aipmodel-0.2.76 → aipmodel-0.2.77}/PKG-INFO +1 -1
- {aipmodel-0.2.76 → aipmodel-0.2.77}/aipmodel/CephS3Manager.py +10 -2
- {aipmodel-0.2.76 → aipmodel-0.2.77}/aipmodel/__init__.py +1 -1
- {aipmodel-0.2.76 → aipmodel-0.2.77}/aipmodel/model_registry.py +38 -0
- {aipmodel-0.2.76 → aipmodel-0.2.77}/aipmodel.egg-info/PKG-INFO +1 -1
- {aipmodel-0.2.76 → aipmodel-0.2.77}/setup.py +1 -1
- {aipmodel-0.2.76 → aipmodel-0.2.77}/MANIFEST.in +0 -0
- {aipmodel-0.2.76 → aipmodel-0.2.77}/README.md +0 -0
- {aipmodel-0.2.76 → aipmodel-0.2.77}/aipmodel/acl_manager.py +0 -0
- {aipmodel-0.2.76 → aipmodel-0.2.77}/aipmodel/exceptions.py +0 -0
- {aipmodel-0.2.76 → aipmodel-0.2.77}/aipmodel/template.py +0 -0
- {aipmodel-0.2.76 → aipmodel-0.2.77}/aipmodel/update_checker.py +0 -0
- {aipmodel-0.2.76 → aipmodel-0.2.77}/aipmodel.egg-info/SOURCES.txt +0 -0
- {aipmodel-0.2.76 → aipmodel-0.2.77}/aipmodel.egg-info/dependency_links.txt +0 -0
- {aipmodel-0.2.76 → aipmodel-0.2.77}/aipmodel.egg-info/requires.txt +0 -0
- {aipmodel-0.2.76 → aipmodel-0.2.77}/aipmodel.egg-info/top_level.txt +0 -0
- {aipmodel-0.2.76 → aipmodel-0.2.77}/requirements.txt +0 -0
- {aipmodel-0.2.76 → aipmodel-0.2.77}/setup.cfg +0 -0
|
@@ -633,8 +633,16 @@ class CephS3Manager:
|
|
|
633
633
|
except _TRANSIENT_TRANSFER_ERRORS as e:
|
|
634
634
|
attempt += 1
|
|
635
635
|
if attempt > self.max_transfer_retries:
|
|
636
|
-
|
|
637
|
-
|
|
636
|
+
# Always log this (not gated by verbose): a transient error that
|
|
637
|
+
# survives every retry is a NETWORK problem, and the user must be
|
|
638
|
+
# able to tell it apart from a Model Registry fault in the task log.
|
|
639
|
+
print(
|
|
640
|
+
f"[SDK_NETWORK_ERROR] {label}: NETWORK/connection failure — this is "
|
|
641
|
+
f"NOT a Model Registry problem. The connection to the storage endpoint "
|
|
642
|
+
f"kept dropping mid-transfer and did not recover after "
|
|
643
|
+
f"{self.max_transfer_retries} automatic retries. Check network "
|
|
644
|
+
f"stability and retry. Underlying: {type(e).__name__}: {e}"
|
|
645
|
+
)
|
|
638
646
|
raise
|
|
639
647
|
delay = min(self.transfer_retry_backoff * (2 ** (attempt - 1)), self.transfer_retry_max_delay)
|
|
640
648
|
if self.verbose:
|
|
@@ -20,6 +20,33 @@ from .CephS3Manager import CephS3Manager
|
|
|
20
20
|
from .acl_manager import AclManager
|
|
21
21
|
from .exceptions import ModelRegistryError, ModelNotFoundError, MetadataError, StorageError, IntegrityDisabledError
|
|
22
22
|
|
|
23
|
+
|
|
24
|
+
def _is_transient_network_error(exc):
|
|
25
|
+
"""
|
|
26
|
+
True when a failure was caused by a transient NETWORK/connection problem
|
|
27
|
+
(dropped connection, mid-stream read break, timeout) rather than a Model
|
|
28
|
+
Registry / storage-content fault. Walks the exception chain and also matches
|
|
29
|
+
the wrapped message, because transfer errors bubble up re-wrapped as
|
|
30
|
+
ValueError ("S3 Transfer failed: ... Connection broken: IncompleteRead ...").
|
|
31
|
+
Used only to add a clear, human-readable "this is the network, not the
|
|
32
|
+
registry" note to the task log — it never changes control flow.
|
|
33
|
+
"""
|
|
34
|
+
_NET_MARKERS = (
|
|
35
|
+
"connection broken", "incompleteread", "responsestreaming",
|
|
36
|
+
"connection reset", "connection aborted", "connection refused",
|
|
37
|
+
"connection was closed", "timed out", "read timeout", "broken pipe",
|
|
38
|
+
"network is unreachable", "eof occurred", "endpointconnection",
|
|
39
|
+
"connectionerror", "name or service not known", "temporary failure in name resolution",
|
|
40
|
+
)
|
|
41
|
+
seen = set()
|
|
42
|
+
e = exc
|
|
43
|
+
while e is not None and id(e) not in seen:
|
|
44
|
+
seen.add(id(e))
|
|
45
|
+
if any(m in str(e).lower() for m in _NET_MARKERS):
|
|
46
|
+
return True
|
|
47
|
+
e = getattr(e, "__cause__", None) or getattr(e, "__context__", None)
|
|
48
|
+
return False
|
|
49
|
+
|
|
23
50
|
load_dotenv()
|
|
24
51
|
|
|
25
52
|
logger = logging.getLogger(__name__)
|
|
@@ -1102,6 +1129,17 @@ class MLOpsManager:
|
|
|
1102
1129
|
return model_id
|
|
1103
1130
|
|
|
1104
1131
|
except (Exception, KeyboardInterrupt) as e:
|
|
1132
|
+
if _is_transient_network_error(e):
|
|
1133
|
+
print("[SDK_NETWORK_ERROR] ============================================================")
|
|
1134
|
+
print("[SDK_NETWORK_ERROR] Model add FAILED because of a NETWORK / connection problem")
|
|
1135
|
+
print("[SDK_NETWORK_ERROR] while transferring the model files.")
|
|
1136
|
+
print("[SDK_NETWORK_ERROR] This is NOT a Model Registry problem — the registry and the")
|
|
1137
|
+
print("[SDK_NETWORK_ERROR] storage are healthy. The connection to the storage/source")
|
|
1138
|
+
print("[SDK_NETWORK_ERROR] endpoint dropped and did not recover after the automatic")
|
|
1139
|
+
print("[SDK_NETWORK_ERROR] retries. Nothing was left half-written (rolled back).")
|
|
1140
|
+
print("[SDK_NETWORK_ERROR] Action: check network stability to the S3 source / Ceph,")
|
|
1141
|
+
print("[SDK_NETWORK_ERROR] then retry the add.")
|
|
1142
|
+
print("[SDK_NETWORK_ERROR] ============================================================")
|
|
1105
1143
|
print(f"[SDK_ERROR] Add model failed: {e}")
|
|
1106
1144
|
if "model_id" in locals():
|
|
1107
1145
|
try:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|