bluer-objects 6.166.1__py3-none-any.whl → 6.170.1__py3-none-any.whl

bluer_objects/__init__.py

@@ -4,7 +4,7 @@ ICON = "🌀"
 
 DESCRIPTION = f"{ICON} Object management in Bash."
 
-VERSION = "6.166.1"
+VERSION = "6.170.1"
 
 REPO_NAME = "bluer-objects"
 

bluer_objects/mlflow/lock/__init__.py

@@ -0,0 +1 @@
+

bluer_objects/mlflow/lock/__main__.py

@@ -0,0 +1,58 @@
+import argparse
+
+from blueness import module
+from blueness.argparse.generic import sys_exit
+
+from bluer_objects import NAME
+from bluer_objects.mlflow.lock.functions import lock, unlock
+from bluer_objects.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+parser = argparse.ArgumentParser(NAME)
+parser.add_argument(
+    "task",
+    type=str,
+    help="lock | unlock",
+)
+parser.add_argument(
+    "--object_name",
+    type=str,
+)
+parser.add_argument(
+    "--lock",
+    type=str,
+    default="lock",
+)
+parser.add_argument(
+    "--timeout",
+    type=int,
+    default=-1,
+    help="in seconds",
+)
+parser.add_argument(
+    "--verbose",
+    type=int,
+    default=1,
+    help="0 | 1",
+)
+args = parser.parse_args()
+
+success = False
+if args.task == "lock":
+    success = lock(
+        object_name=args.object_name,
+        lock_name=args.lock,
+        timeout=args.timeout,
+        verbose=args.verbose == 1,
+    )
+elif args.task == "unlock":
+    success = unlock(
+        object_name=args.object_name,
+        lock_name=args.lock,
+        verbose=args.verbose == 1,
+    )
+else:
+    success = None
+
+sys_exit(logger, NAME, args.task, success)

bluer_objects/mlflow/lock/functions.py

@@ -0,0 +1,121 @@
+import time
+
+from blueness import module
+from bluer_options import string
+
+from bluer_objects import NAME, env
+from bluer_objects.mlflow.tags import get_tags, set_tags
+from bluer_objects.logger import logger
+
+
+NAME = module.name(__file__, NAME)
+
+
+def lock(
+    object_name: str,
+    lock_name: str = "lock",
+    timeout: int = -1,
+    verbose: bool = True,
+) -> bool:
+    logger.info(
+        "{}.lock: {}.{}{}".format(
+            NAME,
+            object_name,
+            lock_name,
+            "" if timeout == -1 else " @ {}".format(string.pretty_duration(timeout)),
+        )
+    )
+
+    start_time = time.time()
+    while True:
+        if timeout > 0 and time.time() - start_time > timeout:
+            if verbose:
+                logger.warning(
+                    "{}.lock: {}.{} timeout.".format(
+                        NAME,
+                        object_name,
+                        lock_name,
+                    )
+                )
+            return False
+
+        success, list_of_tags = get_tags(object_name=object_name)
+        if not success:
+            return False
+
+        lock_value = list_of_tags.get(lock_name, "")
+        if lock_value:
+            if verbose:
+                logger.warning(
+                    "{}.lock: {}.{} is locked by {}.".format(
+                        NAME,
+                        object_name,
+                        lock_name,
+                        lock_value,
+                    )
+                )
+            time.sleep(env.MLFLOW_LOCK_WAIT_FOR_CLEARANCE)
+            continue
+
+        lock_value = string.random()
+        if not set_tags(
+            object_name=object_name,
+            tags={lock_name: lock_value},
+            log=verbose,
+            icon="🔒",
+        ):
+            return False
+
+        time.sleep(env.MLFLOW_LOCK_WAIT_FOR_EXCLUSIVITY)
+
+        success, list_of_tags = get_tags(object_name=object_name)
+        if not success:
+            return False
+
+        lock_value_read = list_of_tags.get(lock_name, "")
+        if lock_value_read != lock_value:
+            if verbose:
+                logger.warning(
+                    "{}.lock: {}.{} is relocked by {} != {}.".format(
+                        NAME,
+                        object_name,
+                        lock_name,
+                        lock_value_read,
+                        lock_value,
+                    )
+                )
+            time.sleep(env.MLFLOW_LOCK_WAIT_FOR_CLEARANCE)
+            continue
+
+        break
+
+    logger.info(
+        "{}.lock: {}.{} is locked by {}.".format(
+            NAME,
+            object_name,
+            lock_name,
+            lock_value,
+        )
+    )
+    return True
+
+
+def unlock(
+    object_name: str,
+    lock_name: str = "lock",
+    verbose: bool = True,
+) -> bool:
+    logger.info(
+        "{}.unlock: {}.{}".format(
+            NAME,
+            object_name,
+            lock_name,
+        )
+    )
+
+    return set_tags(
+        object_name=object_name,
+        tags={lock_name: ""},
+        log=verbose,
+        icon="🔒",
+    )

bluer_objects/storage/s3.py

@@ -16,6 +16,104 @@ from bluer_objects.logger import logger
 class S3Interface(StorageInterface):
     name = "s3"
 
+    def clear(
+        self,
+        do_dryrun: bool = True,
+        log: bool = True,
+    ) -> bool:
+        logger.info(
+            "{}.clear({})".format(
+                self.__class__.__name__,
+                "dryrun" if do_dryrun else "",
+            )
+        )
+
+        try:
+            s3 = boto3.client(
+                "s3",
+                endpoint_url=env.S3_STORAGE_ENDPOINT_URL,
+                aws_access_key_id=env.S3_STORAGE_AWS_ACCESS_KEY_ID,
+                aws_secret_access_key=env.S3_STORAGE_AWS_SECRET_ACCESS_KEY,
+            )
+
+            paginator = s3.get_paginator("list_objects_v2")
+            pages = paginator.paginate(
+                Bucket=env.S3_STORAGE_BUCKET,
+                Prefix="test",
+            )
+        except Exception as e:
+            logger.error(e)
+            return False
+
+        list_of_objects = sorted(
+            list(
+                set(
+                    reduce(
+                        lambda x, y: x + y,
+                        [
+                            [
+                                obj["Key"].split("/", 1)[0]
+                                for obj in page.get("Contents", [])
+                            ]
+                            for page in pages
+                        ],
+                        [],
+                    )
+                )
+            )
+        )
+
+        logger.info(f"{len(list_of_objects)} object(s) to delete.")
+
+        for object_name in tqdm(list_of_objects):
+            if not self.delete(
+                object_name=object_name,
+                do_dryrun=do_dryrun,
+            ):
+                return False
+
+        return True
+
+    def delete(
+        self,
+        object_name: str,
+        do_dryrun: bool = True,
+        log: bool = True,
+    ) -> bool:
+        if log:
+            logger.info(
+                "{}.delete({}){}".format(
+                    self.__class__.__name__,
+                    object_name,
+                    " dryrun" if do_dryrun else "",
+                )
+            )
+        if do_dryrun:
+            return True
+
+        try:
+            s3 = boto3.resource(
+                "s3",
+                endpoint_url=env.S3_STORAGE_ENDPOINT_URL,
+                aws_access_key_id=env.S3_STORAGE_AWS_ACCESS_KEY_ID,
+                aws_secret_access_key=env.S3_STORAGE_AWS_SECRET_ACCESS_KEY,
+            )
+            bucket = s3.Bucket(env.S3_STORAGE_BUCKET)
+
+            objects_to_delete = bucket.objects.filter(Prefix=f"{object_name}/")
+            delete_requests = [{"Key": obj.key} for obj in objects_to_delete]
+
+            if not delete_requests:
+                logger.warning(f"no files found under {object_name}.")
+                return True
+
+            bucket.delete_objects(Delete={"Objects": delete_requests})
+        except Exception as e:
+            logger.error(e)
+            return False
+
+        return True
+
     def download(
         self,
         object_name: str,

{bluer_objects-6.166.1.dist-info → bluer_objects-6.170.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bluer_objects
-Version: 6.166.1
+Version: 6.170.1
 Summary: 🌀 Object management in Bash.
 Home-page: https://github.com/kamangir/bluer-objects
 Author: Arash Abadpour (Kamangir)
@@ -64,6 +64,6 @@ pip install bluer-objects
 
 [![pylint](https://github.com/kamangir/bluer-objects/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/bluer-objects/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/bluer-objects/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/bluer-objects.svg)](https://pypi.org/project/bluer-objects/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/bluer-objects)](https://pypistats.org/packages/bluer-objects)
 
-built by 🌀 [`bluer README`](https://github.com/kamangir/bluer-objects/tree/main/bluer_objects/README), based on 🌀 [`bluer_objects-6.166.1`](https://github.com/kamangir/bluer-objects).
+built by 🌀 [`bluer README`](https://github.com/kamangir/bluer-objects/tree/main/bluer_objects/README), based on 🌀 [`bluer_objects-6.170.1`](https://github.com/kamangir/bluer-objects).
 
 built by 🌀 [`blueness-3.118.1`](https://github.com/kamangir/blueness).

{bluer_objects-6.166.1.dist-info → bluer_objects-6.170.1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-bluer_objects/__init__.py,sha256=s87Zqxb_mTPssWYNG2qn5n-hBwlNCbjUTyzD0p79oak,315
+bluer_objects/__init__.py,sha256=Up7uiCfpaTfzOTrjFhSTVfEQQ9DtDCzeqOFsRvZ45Gw,315
 bluer_objects/__main__.py,sha256=Yqfov833_hJuRne19WrGhT5DWAPtdffpoMxeSXS7EGw,359
 bluer_objects/config.env,sha256=cpmtTuKHIBvVjX30MpAImFgwM0g24qTHwoO-lCJRInA,175
 bluer_objects/env.py,sha256=aImjtNh8NKaua2rpQVFXWp86QF7nY87fDuMOO14O58E,1956
@@ -115,13 +115,16 @@ bluer_objects/mlflow/objects.py,sha256=aXYHLRCTt1FtDQoyPOG-doCFms7qF6Krz00Fg3kzJ
 bluer_objects/mlflow/runs.py,sha256=v1IKRvxbuKkengESnG-xdUXxxNSkALMeBmfMQwrUKSs,2416
 bluer_objects/mlflow/tags.py,sha256=8uBYRrE4weTLrwPqo-c4M21FEVRANf7SGCcxpoCPhuM,2475
 bluer_objects/mlflow/testing.py,sha256=cJH5Ki02fJN_Xos1j9yvwQChXvMkOa9i12vtDKmkbNc,842
+bluer_objects/mlflow/lock/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+bluer_objects/mlflow/lock/__main__.py,sha256=xF_xq2UqAsEohSOHjxaFXaw9KopOEDg6LRDM5a4VAPQ,1138
+bluer_objects/mlflow/lock/functions.py,sha256=MOslqblNAOsRvILzLF4q6m2EAwCk4f4zEWQpsy8lVnM,3045
 bluer_objects/storage/WebDAV.py,sha256=IvYZDST6Cg1rgRhQ5eh3Hrq8V7S3oon9O29MH55QLxA,2922
 bluer_objects/storage/WebDAVrequest.py,sha256=ihWOxP7h28qj0-5AZ0SsERSBrViq9p4iNVeR4pGBWQo,9836
 bluer_objects/storage/WebDAVzip.py,sha256=Sd_rU57pJaRa05sKNQiMr85Bg3w7O5k8N1pSN1u8omA,3872
 bluer_objects/storage/__init__.py,sha256=XosPRjB20wDS-QMwQtr3zqKC9UM7U8z-PHwySGBegcQ,1646
 bluer_objects/storage/__main__.py,sha256=vZI6rUkrekf1eYUgWOOUnFhl4qPfpByzwb-tihTOiIo,1776
 bluer_objects/storage/base.py,sha256=72I1zvlpbQbFU20TxqcTodR4m8PYgBPXMfteek8V0_A,1949
-bluer_objects/storage/s3.py,sha256=Xm0Dj2qQG8oEJyAM91NV9clDxGJxyN3FQughlFvP17w,5460
+bluer_objects/storage/s3.py,sha256=evUFpPk6PUv1KCXTMq_QCbygqNk0YNAOQr_B4gYrjBE,8342
 bluer_objects/testing/__init__.py,sha256=DWY5ZtvCnHG_t9BDiqy_ArLOZi-nlyAtPVMLA1PPAMU,62
 bluer_objects/testing/__main__.py,sha256=hhJV9qn0V_8FxzNDcoHCHr4A7zf9UudnNGJCAPkTBGU,750
 bluer_objects/testing/functions.py,sha256=AXAfzWLcEPkbSYTehdahshjKJ45C4IJkRs_TgrHOntc,1355
@@ -153,8 +156,8 @@ bluer_objects/tests/test_storage_webdav_request.py,sha256=h2b8PeIx0-hQ2d6PmQcJZy
 bluer_objects/tests/test_storage_webdav_zip.py,sha256=C19qxhkcHyTwVFzW35vL85SOcXJPkqXXaWUNll0Uyqc,1017
 bluer_objects/tests/test_testing.py,sha256=d2NH435yqJBl9wmfMqGGd-f0Y0jsL2QhHUXkty9AwPA,235
 bluer_objects/tests/test_version.py,sha256=Lyf3PMcA22e17BNRk_2VgPrtao6dWEgVoXo68Uds8SE,75
-bluer_objects-6.166.1.dist-info/licenses/LICENSE,sha256=ogEPNDSH0_dhiv_lT3ifVIdgIzHAqNA_SemnxUfPBJk,7048
-bluer_objects-6.166.1.dist-info/METADATA,sha256=rzbixpQZNpW0GGeFnFo4fr9ZLyQnHFUEW_veSOp3O6w,3678
-bluer_objects-6.166.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-bluer_objects-6.166.1.dist-info/top_level.txt,sha256=RX2TpddbnRkurda3G_pAdyeTztP2IhhRPx949GlEvQo,14
-bluer_objects-6.166.1.dist-info/RECORD,,
+bluer_objects-6.170.1.dist-info/licenses/LICENSE,sha256=ogEPNDSH0_dhiv_lT3ifVIdgIzHAqNA_SemnxUfPBJk,7048
+bluer_objects-6.170.1.dist-info/METADATA,sha256=ELD0riVQPhS4tFIUH_2uAloiFQl-_KJuL7nc5xozizw,3678
+bluer_objects-6.170.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+bluer_objects-6.170.1.dist-info/top_level.txt,sha256=RX2TpddbnRkurda3G_pAdyeTztP2IhhRPx949GlEvQo,14
+bluer_objects-6.170.1.dist-info/RECORD,,