aipmodel 0.2.74__tar.gz → 0.2.76__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {aipmodel-0.2.74 → aipmodel-0.2.76}/PKG-INFO +1 -1
- {aipmodel-0.2.74 → aipmodel-0.2.76}/README.md +13 -0
- {aipmodel-0.2.74 → aipmodel-0.2.76}/aipmodel/CephS3Manager.py +5 -0
- {aipmodel-0.2.74 → aipmodel-0.2.76}/aipmodel/__init__.py +1 -1
- {aipmodel-0.2.74 → aipmodel-0.2.76}/aipmodel/model_registry.py +40 -6
- {aipmodel-0.2.74 → aipmodel-0.2.76}/aipmodel.egg-info/PKG-INFO +1 -1
- {aipmodel-0.2.74 → aipmodel-0.2.76}/setup.py +1 -1
- {aipmodel-0.2.74 → aipmodel-0.2.76}/MANIFEST.in +0 -0
- {aipmodel-0.2.74 → aipmodel-0.2.76}/aipmodel/acl_manager.py +0 -0
- {aipmodel-0.2.74 → aipmodel-0.2.76}/aipmodel/exceptions.py +0 -0
- {aipmodel-0.2.74 → aipmodel-0.2.76}/aipmodel/template.py +0 -0
- {aipmodel-0.2.74 → aipmodel-0.2.76}/aipmodel/update_checker.py +0 -0
- {aipmodel-0.2.74 → aipmodel-0.2.76}/aipmodel.egg-info/SOURCES.txt +0 -0
- {aipmodel-0.2.74 → aipmodel-0.2.76}/aipmodel.egg-info/dependency_links.txt +0 -0
- {aipmodel-0.2.74 → aipmodel-0.2.76}/aipmodel.egg-info/requires.txt +0 -0
- {aipmodel-0.2.74 → aipmodel-0.2.76}/aipmodel.egg-info/top_level.txt +0 -0
- {aipmodel-0.2.74 → aipmodel-0.2.76}/requirements.txt +0 -0
- {aipmodel-0.2.74 → aipmodel-0.2.76}/setup.cfg +0 -0
|
@@ -44,6 +44,19 @@ CEPH_ADMIN_SECRET_KEY="your_public_bucket_secret_key"
|
|
|
44
44
|
ACL_ENABLED="false"
|
|
45
45
|
ACL_API_HOST="http://your-acl-server:30011"
|
|
46
46
|
|
|
47
|
+
# Where per-user credentials come from. Default true: resolve the user's S3 +
|
|
48
|
+
# Mlops-Core keys from the user-management service using the bearer token.
|
|
49
|
+
# Set false to run WITHOUT user-management (other systems / testing): the SDK
|
|
50
|
+
# then reads the keys straight from the env vars below instead.
|
|
51
|
+
USER_MANAGEMENT_ENABLED="true"
|
|
52
|
+
# Only used when USER_MANAGEMENT_ENABLED="false":
|
|
53
|
+
# CEPH_USER_ACCESS_KEY="..." # the user's S3 access key
|
|
54
|
+
# CEPH_USER_SECRET_KEY="..." # the user's S3 secret key
|
|
55
|
+
# CEPH_USER_BUCKET="..." # the user's bucket
|
|
56
|
+
# MLOPS_CORE_ACCESS_KEY="..." # Mlops-Core basic-auth access key
|
|
57
|
+
# MLOPS_CORE_SECRET_KEY="..." # Mlops-Core basic-auth secret key
|
|
58
|
+
# MLOPS_CORE_USERNAME="..." # display/owner name
|
|
59
|
+
|
|
47
60
|
# Registry<->storage integrity (zombie/orphan) checks - opt-in, default off.
|
|
48
61
|
# When false, verify_model_integrity / verify_all_models / find_orphan_models
|
|
49
62
|
# raise IntegrityDisabledError instead of running the (S3-listing) consistency
|
|
@@ -37,6 +37,11 @@ _TRANSIENT_TRANSFER_ERRORS = tuple(
|
|
|
37
37
|
"EndpointConnectionError",
|
|
38
38
|
"ReadTimeoutError",
|
|
39
39
|
"IncompleteReadError",
|
|
40
|
+
# boto3 wraps a mid-stream connection break (urllib3 ProtocolError /
|
|
41
|
+
# http IncompleteRead) into ResponseStreamingError before it reaches the
|
|
42
|
+
# caller, so the underlying transient types below never match on their
|
|
43
|
+
# own for a download that dies mid-body. Retry on the wrapper too.
|
|
44
|
+
"ResponseStreamingError",
|
|
40
45
|
)
|
|
41
46
|
if hasattr(_botocore_exceptions, _name)
|
|
42
47
|
) + (
|
|
@@ -369,6 +369,7 @@ class MLOpsManager:
|
|
|
369
369
|
ACL_API_HOST=None,
|
|
370
370
|
ACL_ENABLED=None,
|
|
371
371
|
INTEGRITY_ENABLED=None,
|
|
372
|
+
USER_MANAGEMENT_ENABLED=None,
|
|
372
373
|
MLOPS_CORE_API_HOST=None,
|
|
373
374
|
MLOPS_CORE_WEB_HOST=None
|
|
374
375
|
):
|
|
@@ -408,19 +409,52 @@ class MLOpsManager:
|
|
|
408
409
|
|
|
409
410
|
self.USER_MANAGEMENT_API = USER_MANAGEMENT_API or os.environ.get("USER_MANAGEMENT_API")
|
|
410
411
|
self.CEPH_ENDPOINT_URL = CEPH_ENDPOINT_URL or os.environ.get("CEPH_ENDPOINT_URL")
|
|
411
|
-
|
|
412
|
+
|
|
413
|
+
# Where per-user credentials come from. Default True: resolve the user's
|
|
414
|
+
# S3 + Mlops-Core keys from the user-management service via the bearer
|
|
415
|
+
# token (production). Set USER_MANAGEMENT_ENABLED=false to run on a system
|
|
416
|
+
# WITHOUT user-management — the keys are then read straight from env:
|
|
417
|
+
# CEPH_USER_ACCESS_KEY / CEPH_USER_SECRET_KEY / CEPH_USER_BUCKET /
|
|
418
|
+
# MLOPS_CORE_ACCESS_KEY / MLOPS_CORE_SECRET_KEY / MLOPS_CORE_USERNAME.
|
|
419
|
+
self.USER_MANAGEMENT_ENABLED = self._to_bool(
|
|
420
|
+
USER_MANAGEMENT_ENABLED if USER_MANAGEMENT_ENABLED is not None else os.environ.get("USER_MANAGEMENT_ENABLED"),
|
|
421
|
+
default=True
|
|
422
|
+
)
|
|
423
|
+
|
|
412
424
|
# Public Bucket Configuration
|
|
413
425
|
self.CEPH_PUBLIC_BUCKET_NAME = os.environ.get("CEPH_PUBLIC_BUCKET_NAME", "aip-public")
|
|
414
426
|
self.PUBLIC_ACCESS_KEY = os.environ.get("CEPH_ADMIN_ACCESS_KEY")
|
|
415
427
|
self.PUBLIC_SECRET_KEY = os.environ.get("CEPH_ADMIN_SECRET_KEY")
|
|
416
428
|
|
|
417
|
-
|
|
429
|
+
if self.USER_MANAGEMENT_ENABLED:
|
|
430
|
+
user_info, self.MLOPS_CORE_USERNAME = get_user_metadata(bearer_token=user_token, user_management_url=self.USER_MANAGEMENT_API)
|
|
431
|
+
else:
|
|
432
|
+
# Direct-credential mode — no user-management call; keys come from env.
|
|
433
|
+
self.MLOPS_CORE_USERNAME = os.environ.get("MLOPS_CORE_USERNAME") or os.environ.get("CLEARML_USERNAME") or "local"
|
|
434
|
+
user_info = {
|
|
435
|
+
"s3_access_key": os.environ.get("CEPH_USER_ACCESS_KEY") or os.environ.get("CEPH_ADMIN_ACCESS_KEY"),
|
|
436
|
+
"s3_secret_key": os.environ.get("CEPH_USER_SECRET_KEY") or os.environ.get("CEPH_ADMIN_SECRET_KEY"),
|
|
437
|
+
"s3_bucket": os.environ.get("CEPH_USER_BUCKET"),
|
|
438
|
+
"clearml_access_key": os.environ.get("MLOPS_CORE_ACCESS_KEY") or os.environ.get("CLEARML_ACCESS_KEY"),
|
|
439
|
+
"clearml_secret_key": os.environ.get("MLOPS_CORE_SECRET_KEY") or os.environ.get("CLEARML_SECRET_KEY"),
|
|
440
|
+
}
|
|
441
|
+
missing_creds = [k for k, v in user_info.items() if not v]
|
|
442
|
+
if missing_creds:
|
|
443
|
+
raise ValueError(
|
|
444
|
+
"USER_MANAGEMENT_ENABLED is false but direct credentials are missing from env: "
|
|
445
|
+
+ ", ".join(missing_creds)
|
|
446
|
+
+ " (set CEPH_USER_ACCESS_KEY / CEPH_USER_SECRET_KEY / CEPH_USER_BUCKET / "
|
|
447
|
+
"MLOPS_CORE_ACCESS_KEY / MLOPS_CORE_SECRET_KEY)."
|
|
448
|
+
)
|
|
418
449
|
|
|
419
|
-
|
|
450
|
+
# USER_MANAGEMENT_API is only required when that service is actually used.
|
|
451
|
+
required_cfg = {
|
|
420
452
|
"MLOPS_CORE_API_HOST": self.MLOPS_CORE_API_HOST,
|
|
421
|
-
"USER_MANAGEMENT_API": self.USER_MANAGEMENT_API,
|
|
422
453
|
"CEPH_ENDPOINT_URL": self.CEPH_ENDPOINT_URL,
|
|
423
|
-
}
|
|
454
|
+
}
|
|
455
|
+
if self.USER_MANAGEMENT_ENABLED:
|
|
456
|
+
required_cfg["USER_MANAGEMENT_API"] = self.USER_MANAGEMENT_API
|
|
457
|
+
missing_cfg = [k for k, v in required_cfg.items() if not v]
|
|
424
458
|
if missing_cfg:
|
|
425
459
|
error_msg = f"Missing required configuration: {', '.join(missing_cfg)}"
|
|
426
460
|
print(f"[SDK_FAIL] {error_msg}")
|
|
@@ -449,7 +483,7 @@ class MLOpsManager:
|
|
|
449
483
|
if not skip_connection_check:
|
|
450
484
|
if self.verbose:
|
|
451
485
|
print("[SDK_INFO] Performing health checks...")
|
|
452
|
-
if not self.check_user_management_service():
|
|
486
|
+
if self.USER_MANAGEMENT_ENABLED and not self.check_user_management_service():
|
|
453
487
|
raise ValueError("User Management API unreachable")
|
|
454
488
|
if not self.check_mlops_core_service():
|
|
455
489
|
raise ValueError("Mlops-Core Server down")
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|