bluer-objects 6.107.1__tar.gz → 6.133.1__tar.gz

{bluer_objects-6.107.1/bluer_objects.egg-info → bluer_objects-6.133.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bluer_objects
-Version: 6.107.1
+Version: 6.133.1
 Summary: 🌀 Object management in Bash.
 Home-page: https://github.com/kamangir/bluer-objects
 Author: Arash Abadpour (Kamangir)
@@ -63,6 +63,6 @@ pip install bluer-objects
 
 [![pylint](https://github.com/kamangir/bluer-objects/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/bluer-objects/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/bluer-objects/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/bluer-objects.svg)](https://pypi.org/project/bluer-objects/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/bluer-objects)](https://pypistats.org/packages/bluer-objects)
 
-built by 🌀 [`bluer README`](https://github.com/kamangir/bluer-objects/tree/main/bluer_objects/README), based on 🌀 [`bluer_objects-6.107.1`](https://github.com/kamangir/bluer-objects).
+built by 🌀 [`bluer README`](https://github.com/kamangir/bluer-objects/tree/main/bluer_objects/README), based on 🌀 [`bluer_objects-6.133.1`](https://github.com/kamangir/bluer-objects).
 
 built by 🌀 [`blueness-3.118.1`](https://github.com/kamangir/blueness).

{bluer_objects-6.107.1 → bluer_objects-6.133.1}/README.md

@@ -30,4 +30,4 @@ pip install bluer-objects
 
 [![pylint](https://github.com/kamangir/bluer-objects/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/bluer-objects/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/bluer-objects/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/bluer-objects.svg)](https://pypi.org/project/bluer-objects/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/bluer-objects)](https://pypistats.org/packages/bluer-objects)
 
-built by 🌀 [`bluer README`](https://github.com/kamangir/bluer-objects/tree/main/bluer_objects/README), based on 🌀 [`bluer_objects-6.107.1`](https://github.com/kamangir/bluer-objects).
+built by 🌀 [`bluer README`](https://github.com/kamangir/bluer-objects/tree/main/bluer_objects/README), based on 🌀 [`bluer_objects-6.133.1`](https://github.com/kamangir/bluer-objects).

{bluer_objects-6.107.1 → bluer_objects-6.133.1}/bluer_objects/.abcli/tests/storage.sh

@@ -11,10 +11,13 @@ function test_bluer_objects_storage() {
         create_test_asset \
         --object_name $object_name
     [[ $? -ne 0 ]] && return 1
+    bluer_ai_hr
 
-    # testing upload
+    bluer_objects_ls local $object_name
+    [[ $? -ne 0 ]] && return 1
     bluer_ai_hr
 
+    # upload
     bluer_objects_upload \
         filename=this.yaml \
         $object_name
@@ -33,12 +36,15 @@ function test_bluer_objects_storage() {
     [[ $? -ne 0 ]] && return 1
     bluer_ai_hr
 
+    bluer_objects_ls cloud $object_name
+    [[ $? -ne 0 ]] && return 1
+    bluer_ai_hr
+
     # clean-up
     rm -rfv $object_path
     bluer_ai_hr
 
-    # testing download
-
+    # download
     bluer_objects_download \
         filename=this.yaml \
         $object_name

{bluer_objects-6.107.1 → bluer_objects-6.133.1}/bluer_objects/__init__.py

@@ -4,7 +4,7 @@ ICON = "🌀"
 
 DESCRIPTION = f"{ICON} Object management in Bash."
 
-VERSION = "6.107.1"
+VERSION = "6.133.1"
 
 REPO_NAME = "bluer-objects"
 

bluer_objects-6.133.1/bluer_objects/config.env

@@ -0,0 +1,3 @@
+ABCLI_MLFLOW_EXPERIMENT_PREFIX=experiment/
+
+BLUER_OBJECTS_STORAGE_INTERFACE=webdav-request

{bluer_objects-6.107.1 → bluer_objects-6.133.1}/bluer_objects/file/functions.py

@@ -150,6 +150,8 @@ def download(
         return True
 
     try:
+        os.makedirs(path(filename), exist_ok=True)
+
         # https://stackoverflow.com/a/27406501
         with urllib3.PoolManager().request(
             "GET", url, preload_content=False

{bluer_objects-6.107.1 → bluer_objects-6.133.1}/bluer_objects/storage/WebDAV.py

@@ -82,13 +82,6 @@ class WebDAVInterface(StorageInterface):
             log=log,
         )
 
-    def ls(
-        self,
-        object_name: str,
-        where: str = "local",
-    ) -> Tuple[bool, List[str]]:
-        assert False, "not implemented"
-
     def upload(
         self,
         object_name: str,

bluer_objects-6.133.1/bluer_objects/storage/WebDAVrequest.py

@@ -0,0 +1,346 @@
+import requests
+from requests.auth import HTTPBasicAuth
+import glob
+from typing import Tuple, List
+from xml.etree import ElementTree as ET
+from tqdm import tqdm
+
+from bluer_objects.storage.base import StorageInterface
+from bluer_objects import env, file, path
+from bluer_objects import objects
+from bluer_objects.logger import logger
+
+
+# https://chatgpt.com/c/6824cf43-6738-8005-8733-54b6a77f20ee
+class WebDAVRequestInterface(StorageInterface):
+    name = "webdav-request"
+
+    def clear(
+        self,
+        do_dryrun: bool = True,
+        log: bool = True,
+    ) -> bool:
+        logger.info(
+            "{}.clear({})".format(
+                self.__class__.__name__,
+                "dryrun" if do_dryrun else "",
+            )
+        )
+
+        success, list_of_objects = self.list_raw(
+            suffix="",
+            recursive=False,
+        )
+        if not success:
+            return False
+
+        count: int = 0
+        for thing in tqdm(list_of_objects):
+            thing_name = thing.split(f"{env.WEBDAV_LOGIN}/", 1)[1]
+            if not thing_name.startswith("test"):
+                continue
+
+            if not thing_name.endswith("/"):
+                continue
+
+            if not self.delete(
+                object_name=thing_name.split("/", 1)[0],
+                do_dryrun=do_dryrun,
+                log=log,
+            ):
+                return False
+
+            count += 1
+
+        logger.info(f"deleted {count} object(s).")
+        return True
+
+    def delete(
+        self,
+        object_name: str,
+        do_dryrun: bool = True,
+        log: bool = True,
+    ) -> bool:
+        if log:
+            logger.info(
+                "{}.delete({}){}".format(
+                    self.__class__.__name__,
+                    object_name,
+                    " dryrun" if do_dryrun else "",
+                )
+            )
+        if do_dryrun:
+            return True
+
+        try:
+            response = requests.request(
+                method="DELETE",
+                url=f"{env.WEBDAV_HOSTNAME}/{object_name}/",
+                auth=HTTPBasicAuth(
+                    env.WEBDAV_LOGIN,
+                    env.WEBDAV_PASSWORD,
+                ),
+            )
+        except Exception as e:
+            logger.error(e)
+            return False
+
+        if response.status_code in [200, 204]:
+            return True
+
+        logger.error(f"failed to delete: {response.status_code} - {response.text}")
+        return False
+
+    def mkdir(
+        self,
+        path: str,
+        log: bool = True,
+    ) -> bool:
+        url = f"{env.WEBDAV_HOSTNAME}/"
+        for folder in path.split("/"):
+            url = f"{url}{folder}/"
+
+            try:
+                response = requests.request(
+                    "MKCOL",
+                    url,
+                    auth=HTTPBasicAuth(
+                        env.WEBDAV_LOGIN,
+                        env.WEBDAV_PASSWORD,
+                    ),
+                )
+            except Exception as e:
+                logger.error(e)
+                return False
+
+            if response.status_code == 405:  # Already exists
+                continue
+
+            if response.status_code == 201:  # Created
+                if log:
+                    logger.info(
+                        "{}.mkdir {}".format(
+                            self.__class__.__name__,
+                            url.split(env.WEBDAV_HOSTNAME, 1)[1],
+                        )
+                    )
+                continue
+
+            logger.error(f"failed to create: {response.status_code} - {response.text}")
+            return False
+
+        return True
+
+    def download(
+        self,
+        object_name: str,
+        filename: str = "",
+        log: bool = True,
+    ) -> bool:
+        if filename:
+            local_path = objects.path_of(
+                object_name=object_name,
+                filename=filename,
+                create=True,
+            )
+
+            if not path.create(file.path(local_path)):
+                return False
+
+            url = f"{env.WEBDAV_HOSTNAME}/{object_name}/{filename}"
+
+            try:
+                response = requests.get(
+                    url,
+                    auth=HTTPBasicAuth(
+                        env.WEBDAV_LOGIN,
+                        env.WEBDAV_PASSWORD,
+                    ),
+                )
+            except Exception as e:
+                logger.error(e)
+                return False
+
+            if response.status_code == 404:  # object not found
+                return True
+
+            if response.status_code == 200:
+                try:
+                    with open(local_path, "wb") as file_:
+                        file_.write(response.content)
+                except Exception as e:
+                    logger.error(e)
+                    return False
+
+                return super().download(
+                    object_name=object_name,
+                    filename=filename,
+                    log=log,
+                )
+
+            logger.error(f"failed to download: {response.status_code}")
+            return False
+
+        success, list_of_files = self.ls(
+            object_name=object_name,
+            where="cloud",
+        )
+        if not success:
+            return False
+
+        for filename_ in tqdm(list_of_files):
+            if not self.download(
+                object_name=object_name,
+                filename=filename_,
+                log=log,
+            ):
+                return False
+
+        return True
+
+    def list_raw(
+        self,
+        suffix: str,
+        recursive: bool,
+    ) -> Tuple[bool, List[str]]:
+        # https://chatgpt.com/c/6824f8d3-d9c0-8005-a7fa-d646f812f4b7
+        headers = {
+            "Depth": "infinity" if recursive else "1",
+            "Content-Type": "application/xml",
+        }
+
+        # Minimal PROPFIND XML body
+        data = """<?xml version="1.0"?>
+        <d:propfind xmlns:d="DAV:">
+        <d:prop><d:displayname/></d:prop>
+        </d:propfind>"""
+
+        try:
+            response = requests.request(
+                method="PROPFIND",
+                url=f"{env.WEBDAV_HOSTNAME}/{suffix}",
+                data=data,
+                headers=headers,
+                auth=HTTPBasicAuth(
+                    env.WEBDAV_LOGIN,
+                    env.WEBDAV_PASSWORD,
+                ),
+            )
+        except Exception as e:
+            logger.error(e)
+            return False, []
+
+        if response.status_code == 404:  # object not found
+            return True, []
+
+        if response.status_code in (207, 207):
+            tree = ET.fromstring(response.content)
+            ns = {"d": "DAV:"}
+            list_of_files = []
+            for resp in tree.findall("d:response", ns):
+                href = resp.find("d:href", ns).text
+                list_of_files.append(href)
+
+            return True, list_of_files
+
+        logger.error(f"failed to list: {response.status_code} - {response.text}")
+        return False, []
+
+    def ls(
+        self,
+        object_name: str,
+        where: str = "local",
+    ) -> Tuple[bool, List[str]]:
+        if where == "cloud":
+            success, list_of_files = self.list_raw(
+                suffix=f"{object_name}/",
+                recursive=True,
+            )
+
+            return success, sorted(
+                [
+                    filename
+                    for filename in [
+                        filename.split(f"{env.WEBDAV_LOGIN}/{object_name}/", 1)[1]
+                        for filename in list_of_files
+                        if not filename.endswith("/")
+                    ]
+                    if filename
+                ]
+            )
+
+        return super().ls(
+            object_name=object_name,
+            where=where,
+        )
+
+    def upload(
+        self,
+        object_name: str,
+        filename: str = "",
+        log: bool = True,
+    ) -> bool:
+        if filename:
+            if not self.mkdir(
+                path="{}/{}".format(
+                    object_name,
+                    file.path(filename),
+                ),
+                log=log,
+            ):
+                return False
+
+            url = f"{env.WEBDAV_HOSTNAME}/{object_name}/{filename}"
+
+            local_path = objects.path_of(
+                object_name=object_name,
+                filename=filename,
+            )
+
+            try:
+                with open(local_path, "rb") as file_data:
+                    response = requests.put(
+                        url,
+                        data=file_data,
+                        auth=HTTPBasicAuth(
+                            env.WEBDAV_LOGIN,
+                            env.WEBDAV_PASSWORD,
+                        ),
+                    )
+            except Exception as e:
+                logger.error(e)
+                return False
+
+            if response.status_code in [200, 201, 204]:
+                return super().upload(
+                    object_name=object_name,
+                    filename=filename,
+                    log=log,
+                )
+
+            logger.error(f"failed to upload: {response.status_code} - {response.text}")
+            return False
+
+        object_path = "{}/".format(objects.object_path(object_name=object_name))
+        for filename_ in tqdm(
+            sorted(
+                glob.glob(
+                    objects.path_of(
+                        object_name=object_name,
+                        filename="**",
+                    ),
+                    recursive=True,
+                )
+            )
+        ):
+            if not file.exists(filename_):
+                continue
+
+            if not self.upload(
+                object_name=object_name,
+                filename=filename_.split(object_path, 1)[1],
+                log=log,
+            ):
+                return False
+
+        return True

{bluer_objects-6.107.1 → bluer_objects-6.133.1}/bluer_objects/storage/WebDAVzip.py

@@ -13,7 +13,7 @@ from bluer_objects.logger import logger
 
 # tars the objects to avoid 'content-length' - see WebDAVInterface.
 class WebDAVzipInterface(StorageInterface):
-    name = "webdavzip"
+    name = "webdav-zip"
 
     def __init__(self):
         super().__init__()
@@ -106,24 +106,6 @@ class WebDAVzipInterface(StorageInterface):
         object_name: str,
         where: str = "local",
     ) -> Tuple[bool, List[str]]:
-        if where == "local":
-            object_path = objects.object_path(
-                object_name=object_name,
-            )
-
-            return True, [
-                os.path.relpath(filename, start=object_path)
-                for filename in glob.glob(
-                    os.path.join(
-                        object_path,
-                        "**",
-                        "*",
-                    ),
-                    recursive=True,
-                )
-                if os.path.isfile(filename)
-            ]
-
         if where == "cloud":
             try:
                 if self.client.check(remote_path=f"{object_name}.zip"):
@@ -134,8 +116,10 @@ class WebDAVzipInterface(StorageInterface):
 
             return True, []
 
-        logger.error(f"Unknown 'where': {where}")
-        return False, []
+        return super().ls(
+            object_name=object_name,
+            where=where,
+        )
 
     def upload(
         self,

{bluer_objects-6.107.1 → bluer_objects-6.133.1}/bluer_objects/storage/__init__.py

@@ -2,6 +2,7 @@ from typing import Tuple, List
 
 from bluer_objects.storage.base import StorageInterface
 from bluer_objects.storage.WebDAV import WebDAVInterface
+from bluer_objects.storage.WebDAVrequest import WebDAVRequestInterface
 from bluer_objects.storage.WebDAVzip import WebDAVzipInterface
 from bluer_objects import env
 from bluer_objects.logger import logger
@@ -10,6 +11,8 @@ interface = StorageInterface()
 
 if env.BLUER_OBJECTS_STORAGE_INTERFACE == WebDAVInterface.name:
     interface = WebDAVInterface()
+elif env.BLUER_OBJECTS_STORAGE_INTERFACE == WebDAVRequestInterface.name:
+    interface = WebDAVRequestInterface()
 elif env.BLUER_OBJECTS_STORAGE_INTERFACE == WebDAVzipInterface.name:
     interface = WebDAVzipInterface()
 else:

{bluer_objects-6.107.1 → bluer_objects-6.133.1}/bluer_objects/storage/base.py

@@ -1,5 +1,8 @@
+import os
+import glob
 from typing import Tuple, List
 
+from bluer_objects import objects
 from bluer_objects.logger import logger
 
 
@@ -8,7 +11,7 @@ class StorageInterface:
         self,
         do_dryrun: bool = True,
     ) -> bool:
-        return False
+        return True
 
     def download(
         self,
@@ -32,7 +35,32 @@ class StorageInterface:
         object_name: str,
         where: str = "local",
     ) -> Tuple[bool, List[str]]:
-        return True, []
+        if where == "local":
+            object_path = objects.object_path(
+                object_name=object_name,
+            )
+
+            return True, sorted(
+                [
+                    os.path.relpath(filename, start=object_path)
+                    for filename in glob.glob(
+                        os.path.join(
+                            object_path,
+                            "**",
+                            "*",
+                        ),
+                        recursive=True,
+                    )
+                    if os.path.isfile(filename)
+                ]
+            )
+
+        if where == "cloud":
+            logger.error("not implemented")
+            return False, []
+
+        logger.error(f"Unknown 'where': {where}")
+        return False, []
 
     def upload(
         self,

{bluer_objects-6.107.1 → bluer_objects-6.133.1}/bluer_objects/tests/test_env.py

@@ -2,6 +2,7 @@ from bluer_ai.tests.test_env import test_bluer_ai_env
 
 from bluer_objects import env
 from bluer_objects.storage.WebDAV import WebDAVInterface
+from bluer_objects.storage.WebDAVrequest import WebDAVRequestInterface
 from bluer_objects.storage.WebDAVzip import WebDAVzipInterface
 
 
@@ -14,6 +15,7 @@ def test_bluer_objects_env():
 
     assert env.BLUER_OBJECTS_STORAGE_INTERFACE in [
         WebDAVInterface.name,
+        WebDAVRequestInterface.name,
         WebDAVzipInterface.name,
     ]
 

bluer_objects-6.133.1/bluer_objects/tests/test_file_download.py

@@ -0,0 +1,25 @@
+import pytest
+
+from bluer_objects import objects, file
+
+
+@pytest.mark.parametrize(
+    ["url"],
+    [
+        ["https://raw.githubusercontent.com/jbrownlee/Datasets/master/shampoo.csv"],
+    ],
+)
+def test_file_download(url: str):
+    object_name = objects.unique_object("test_file_download")
+
+    filename = objects.path_of(
+        object_name=object_name,
+        filename=url.split("/")[-1],
+    )
+
+    assert file.download(
+        url=url,
+        filename=filename,
+    )
+
+    assert file.exists(filename)

{bluer_objects-6.107.1 → bluer_objects-6.133.1}/bluer_objects/tests/test_file_load_save.py

@@ -94,7 +94,7 @@ def test_file_load_save_matrix(
     size: Tuple[int, ...],
     dtype: Union[np.dtype, type],
 ) -> None:
-    object_name = objects.unique_object(test_file_load_save_matrix)
+    object_name = objects.unique_object("test_file_load_save_matrix")
 
     test_matrix = (
         np.random.randint(0, 256, size=size, dtype=dtype)

{bluer_objects-6.107.1 → bluer_objects-6.133.1}/bluer_objects/tests/test_logger_matrix.py

@@ -18,7 +18,7 @@ def test_log_matrix(
     test_image,
     verbose: bool,
 ):
-    object_name = objects.unique_object()
+    object_name = objects.unique_object("test_log_matrix")
 
     assert log_matrix(
         matrix=test_image,
@@ -58,7 +58,7 @@ def test_log_matrix_hist(
     test_image,
     verbose: bool,
 ):
-    object_name = objects.unique_object()
+    object_name = objects.unique_object("test_log_matrix_hist")
 
     assert log_matrix_hist(
         matrix=test_image,

{bluer_objects-6.107.1 → bluer_objects-6.133.1}/bluer_objects/tests/test_metadata.py

@@ -77,7 +77,8 @@ def test_metadata(
     get_source: Callable[[str], str],
     get_source_type: MetadataSourceType,
 ):
-    object_name = unique_object()
+    object_name = unique_object("test_metadata")
+
     key = random()
     value = random()
 
@@ -98,7 +99,8 @@ def test_metadata(
 
 
 def test_metadata_dict():
-    object_name = unique_object()
+    object_name = unique_object("test_metadata_dict")
+
     key = random()
     value = {random(): random() for _ in range(10)}
 
@@ -138,7 +140,8 @@ def test_metadata_dict():
 
 
 def test_metadata_file():
-    object_name = unique_object()
+    object_name = unique_object("test_metadata_file")
+
     key = random()
     value = random()
 
@@ -153,7 +156,8 @@ def test_metadata_file():
 
 
 def test_metadata_object():
-    object_name = unique_object()
+    object_name = unique_object("test_metadata_object")
+
     key = random()
     value = random()
 
@@ -163,7 +167,8 @@ def test_metadata_object():
 
 
 def test_metadata_path():
-    object_name = unique_object()
+    object_name = unique_object("test_metadata_path")
+
     key = random()
     value = random()
 
@@ -175,7 +180,8 @@ def test_metadata_path():
 
 
 def test_metadata_upload():
-    object_name = unique_object()
+    object_name = unique_object("test_metadata_upload")
+
     key = random()
     value = random()