bluer-objects 6.104.1__tar.gz → 6.129.1__tar.gz

{bluer_objects-6.104.1/bluer_objects.egg-info → bluer_objects-6.129.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bluer_objects
-Version: 6.104.1
+Version: 6.129.1
 Summary: 🌀 Object management in Bash.
 Home-page: https://github.com/kamangir/bluer-objects
 Author: Arash Abadpour (Kamangir)
@@ -63,6 +63,6 @@ pip install bluer-objects
 
 [![pylint](https://github.com/kamangir/bluer-objects/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/bluer-objects/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/bluer-objects/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/bluer-objects.svg)](https://pypi.org/project/bluer-objects/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/bluer-objects)](https://pypistats.org/packages/bluer-objects)
 
-built by 🌀 [`bluer README`](https://github.com/kamangir/bluer-objects/tree/main/bluer_objects/README), based on 🌀 [`bluer_objects-6.104.1`](https://github.com/kamangir/bluer-objects).
+built by 🌀 [`bluer README`](https://github.com/kamangir/bluer-objects/tree/main/bluer_objects/README), based on 🌀 [`bluer_objects-6.129.1`](https://github.com/kamangir/bluer-objects).
 
 built by 🌀 [`blueness-3.118.1`](https://github.com/kamangir/blueness).

{bluer_objects-6.104.1 → bluer_objects-6.129.1}/README.md

@@ -30,4 +30,4 @@ pip install bluer-objects
 
 [![pylint](https://github.com/kamangir/bluer-objects/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/bluer-objects/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/bluer-objects/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/bluer-objects.svg)](https://pypi.org/project/bluer-objects/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/bluer-objects)](https://pypistats.org/packages/bluer-objects)
 
-built by 🌀 [`bluer README`](https://github.com/kamangir/bluer-objects/tree/main/bluer_objects/README), based on 🌀 [`bluer_objects-6.104.1`](https://github.com/kamangir/bluer-objects).
+built by 🌀 [`bluer README`](https://github.com/kamangir/bluer-objects/tree/main/bluer_objects/README), based on 🌀 [`bluer_objects-6.129.1`](https://github.com/kamangir/bluer-objects).

{bluer_objects-6.104.1 → bluer_objects-6.129.1}/bluer_objects/.abcli/tests/storage.sh

@@ -11,10 +11,13 @@ function test_bluer_objects_storage() {
         create_test_asset \
         --object_name $object_name
     [[ $? -ne 0 ]] && return 1
+    bluer_ai_hr
 
-    # testing upload
+    bluer_objects_ls local $object_name
+    [[ $? -ne 0 ]] && return 1
     bluer_ai_hr
 
+    # upload
     bluer_objects_upload \
         filename=this.yaml \
         $object_name
@@ -33,12 +36,15 @@ function test_bluer_objects_storage() {
     [[ $? -ne 0 ]] && return 1
     bluer_ai_hr
 
+    bluer_objects_ls cloud $object_name
+    [[ $? -ne 0 ]] && return 1
+    bluer_ai_hr
+
     # clean-up
     rm -rfv $object_path
     bluer_ai_hr
 
-    # testing download
-
+    # download
     bluer_objects_download \
         filename=this.yaml \
         $object_name

{bluer_objects-6.104.1 → bluer_objects-6.129.1}/bluer_objects/__init__.py

@@ -4,7 +4,7 @@ ICON = "🌀"
 
 DESCRIPTION = f"{ICON} Object management in Bash."
 
-VERSION = "6.104.1"
+VERSION = "6.129.1"
 
 REPO_NAME = "bluer-objects"
 

bluer_objects-6.129.1/bluer_objects/config.env

@@ -0,0 +1,3 @@
+ABCLI_MLFLOW_EXPERIMENT_PREFIX=experiment/
+
+BLUER_OBJECTS_STORAGE_INTERFACE=webdav-request

{bluer_objects-6.104.1 → bluer_objects-6.129.1}/bluer_objects/file/load.py

@@ -149,21 +149,14 @@ def load_matrix(
 def load_text(
     filename,
     ignore_error=False,
-    count=-1,
     log=False,
 ) -> Tuple[bool, List[str]]:
     success = True
     text = []
 
     try:
-        if count == -1:
-            with open(filename, "r") as fp:
-                text = fp.read()
-            text = text.split("\n")
-        else:
-            # https://stackoverflow.com/a/1767589/10917551
-            with open(filename) as fp:
-                text = [next(fp) for _ in range(count)]
+        with open(filename, "r") as fp:
+            text = fp.read().splitlines()
     except:
         success = False
         if not ignore_error:

{bluer_objects-6.104.1 → bluer_objects-6.129.1}/bluer_objects/file/save.py

@@ -13,7 +13,6 @@ from bluer_options.host import is_jupyter
 from bluer_objects import NAME
 from bluer_objects.file.classes import JsonEncoder
 from bluer_objects.file.functions import path as file_path
-from bluer_objects.file.load import load_text
 from bluer_objects.path import create as path_create
 from bluer_objects.logger import logger
 
@@ -214,32 +213,15 @@ def save_matrix(
 def save_text(
     filename: str,
     text: List[str],
-    if_different: bool = False,
     log: bool = False,
-    remove_empty_lines: bool = False,
 ) -> bool:
-    if remove_empty_lines:
-        text = [
-            line
-            for line, next_line in zip(text, text[1:] + ["x"])
-            if line.strip() or next_line.strip()
-        ]
-
-    if if_different:
-        _, content = load_text(filename, ignore_error=True)
-
-        if "|".join([line for line in content if line]) == "|".join(
-            [line for line in text if line]
-        ):
-            return True
-
     if not prepare_for_saving(filename):
         return False
 
     success = True
     try:
         with open(filename, "w") as fp:
-            fp.writelines([string + "\n" for string in text])
+            fp.writelines(string + "\n" for string in text)
     except:
         success = False
 

{bluer_objects-6.104.1 → bluer_objects-6.129.1}/bluer_objects/storage/WebDAV.py

@@ -82,13 +82,6 @@ class WebDAVInterface(StorageInterface):
             log=log,
         )
 
-    def ls(
-        self,
-        object_name: str,
-        where: str = "local",
-    ) -> Tuple[bool, List[str]]:
-        assert False, "not implemented"
-
     def upload(
         self,
         object_name: str,

bluer_objects-6.129.1/bluer_objects/storage/WebDAVrequest.py

@@ -0,0 +1,346 @@
+import requests
+from requests.auth import HTTPBasicAuth
+import glob
+from typing import Tuple, List
+from xml.etree import ElementTree as ET
+from tqdm import tqdm
+
+from bluer_objects.storage.base import StorageInterface
+from bluer_objects import env, file, path
+from bluer_objects import objects
+from bluer_objects.logger import logger
+
+
+# https://chatgpt.com/c/6824cf43-6738-8005-8733-54b6a77f20ee
+class WebDAVRequestInterface(StorageInterface):
+    name = "webdav-request"
+
+    def clear(
+        self,
+        do_dryrun: bool = True,
+        log: bool = True,
+    ) -> bool:
+        logger.info(
+            "{}.clear({})".format(
+                self.__class__.__name__,
+                "dryrun" if do_dryrun else "",
+            )
+        )
+
+        success, list_of_objects = self.list_raw(
+            suffix="",
+            recursive=False,
+        )
+        if not success:
+            return False
+
+        count: int = 0
+        for thing in tqdm(list_of_objects):
+            thing_name = thing.split(f"{env.WEBDAV_LOGIN}/", 1)[1]
+            if not thing_name.startswith("test"):
+                continue
+
+            if not thing_name.endswith("/"):
+                continue
+
+            if not self.delete(
+                object_name=thing_name.split("/", 1)[0],
+                do_dryrun=do_dryrun,
+                log=log,
+            ):
+                return False
+
+            count += 1
+
+        logger.info(f"deleted {count} object(s).")
+        return True
+
+    def delete(
+        self,
+        object_name: str,
+        do_dryrun: bool = True,
+        log: bool = True,
+    ) -> bool:
+        if log:
+            logger.info(
+                "{}.delete({}){}".format(
+                    self.__class__.__name__,
+                    object_name,
+                    " dryrun" if do_dryrun else "",
+                )
+            )
+        if do_dryrun:
+            return True
+
+        try:
+            response = requests.request(
+                method="DELETE",
+                url=f"{env.WEBDAV_HOSTNAME}/{object_name}/",
+                auth=HTTPBasicAuth(
+                    env.WEBDAV_LOGIN,
+                    env.WEBDAV_PASSWORD,
+                ),
+            )
+        except Exception as e:
+            logger.error(e)
+            return False
+
+        if response.status_code in [200, 204]:
+            return True
+
+        logger.error(f"failed to delete: {response.status_code} - {response.text}")
+        return False
+
+    def mkdir(
+        self,
+        path: str,
+        log: bool = True,
+    ) -> bool:
+        url = f"{env.WEBDAV_HOSTNAME}/"
+        for folder in path.split("/"):
+            url = f"{url}{folder}/"
+
+            try:
+                response = requests.request(
+                    "MKCOL",
+                    url,
+                    auth=HTTPBasicAuth(
+                        env.WEBDAV_LOGIN,
+                        env.WEBDAV_PASSWORD,
+                    ),
+                )
+            except Exception as e:
+                logger.error(e)
+                return False
+
+            if response.status_code == 405:  # Already exists
+                continue
+
+            if response.status_code == 201:  # Created
+                if log:
+                    logger.info(
+                        "{}.mkdir {}".format(
+                            self.__class__.__name__,
+                            url.split(env.WEBDAV_HOSTNAME, 1)[1],
+                        )
+                    )
+                continue
+
+            logger.error(f"failed to create: {response.status_code} - {response.text}")
+            return False
+
+        return True
+
+    def download(
+        self,
+        object_name: str,
+        filename: str = "",
+        log: bool = True,
+    ) -> bool:
+        if filename:
+            local_path = objects.path_of(
+                object_name=object_name,
+                filename=filename,
+                create=True,
+            )
+
+            if not path.create(file.path(local_path)):
+                return False
+
+            url = f"{env.WEBDAV_HOSTNAME}/{object_name}/{filename}"
+
+            try:
+                response = requests.get(
+                    url,
+                    auth=HTTPBasicAuth(
+                        env.WEBDAV_LOGIN,
+                        env.WEBDAV_PASSWORD,
+                    ),
+                )
+            except Exception as e:
+                logger.error(e)
+                return False
+
+            if response.status_code == 404:  # object not found
+                return True
+
+            if response.status_code == 200:
+                try:
+                    with open(local_path, "wb") as file_:
+                        file_.write(response.content)
+                except Exception as e:
+                    logger.error(e)
+                    return False
+
+                return super().download(
+                    object_name=object_name,
+                    filename=filename,
+                    log=log,
+                )
+
+            logger.error(f"failed to download: {response.status_code}")
+            return False
+
+        success, list_of_files = self.ls(
+            object_name=object_name,
+            where="cloud",
+        )
+        if not success:
+            return False
+
+        for filename_ in tqdm(list_of_files):
+            if not self.download(
+                object_name=object_name,
+                filename=filename_,
+                log=log,
+            ):
+                return False
+
+        return True
+
+    def list_raw(
+        self,
+        suffix: str,
+        recursive: bool,
+    ) -> Tuple[bool, List[str]]:
+        # https://chatgpt.com/c/6824f8d3-d9c0-8005-a7fa-d646f812f4b7
+        headers = {
+            "Depth": "infinity" if recursive else "1",
+            "Content-Type": "application/xml",
+        }
+
+        # Minimal PROPFIND XML body
+        data = """<?xml version="1.0"?>
+        <d:propfind xmlns:d="DAV:">
+        <d:prop><d:displayname/></d:prop>
+        </d:propfind>"""
+
+        try:
+            response = requests.request(
+                method="PROPFIND",
+                url=f"{env.WEBDAV_HOSTNAME}/{suffix}",
+                data=data,
+                headers=headers,
+                auth=HTTPBasicAuth(
+                    env.WEBDAV_LOGIN,
+                    env.WEBDAV_PASSWORD,
+                ),
+            )
+        except Exception as e:
+            logger.error(e)
+            return False, []
+
+        if response.status_code == 404:  # object not found
+            return True, []
+
+        if response.status_code in (207, 207):
+            tree = ET.fromstring(response.content)
+            ns = {"d": "DAV:"}
+            list_of_files = []
+            for resp in tree.findall("d:response", ns):
+                href = resp.find("d:href", ns).text
+                list_of_files.append(href)
+
+            return True, list_of_files
+
+        logger.error(f"failed to list: {response.status_code} - {response.text}")
+        return False, []
+
+    def ls(
+        self,
+        object_name: str,
+        where: str = "local",
+    ) -> Tuple[bool, List[str]]:
+        if where == "cloud":
+            success, list_of_files = self.list_raw(
+                suffix=f"{object_name}/",
+                recursive=True,
+            )
+
+            return success, sorted(
+                [
+                    filename
+                    for filename in [
+                        filename.split(f"{env.WEBDAV_LOGIN}/{object_name}/", 1)[1]
+                        for filename in list_of_files
+                        if not filename.endswith("/")
+                    ]
+                    if filename
+                ]
+            )
+
+        return super().ls(
+            object_name=object_name,
+            where=where,
+        )
+
+    def upload(
+        self,
+        object_name: str,
+        filename: str = "",
+        log: bool = True,
+    ) -> bool:
+        if filename:
+            if not self.mkdir(
+                path="{}/{}".format(
+                    object_name,
+                    file.path(filename),
+                ),
+                log=log,
+            ):
+                return False
+
+            url = f"{env.WEBDAV_HOSTNAME}/{object_name}/{filename}"
+
+            local_path = objects.path_of(
+                object_name=object_name,
+                filename=filename,
+            )
+
+            try:
+                with open(local_path, "rb") as file_data:
+                    response = requests.put(
+                        url,
+                        data=file_data,
+                        auth=HTTPBasicAuth(
+                            env.WEBDAV_LOGIN,
+                            env.WEBDAV_PASSWORD,
+                        ),
+                    )
+            except Exception as e:
+                logger.error(e)
+                return False
+
+            if response.status_code in [200, 201, 204]:
+                return super().upload(
+                    object_name=object_name,
+                    filename=filename,
+                    log=log,
+                )
+
+            logger.error(f"failed to upload: {response.status_code} - {response.text}")
+            return False
+
+        object_path = "{}/".format(objects.object_path(object_name=object_name))
+        for filename_ in tqdm(
+            sorted(
+                glob.glob(
+                    objects.path_of(
+                        object_name=object_name,
+                        filename="**",
+                    ),
+                    recursive=True,
+                )
+            )
+        ):
+            if not file.exists(filename_):
+                continue
+
+            if not self.upload(
+                object_name=object_name,
+                filename=filename_.split(object_path, 1)[1],
+                log=log,
+            ):
+                return False
+
+        return True

{bluer_objects-6.104.1 → bluer_objects-6.129.1}/bluer_objects/storage/WebDAVzip.py

@@ -13,7 +13,7 @@ from bluer_objects.logger import logger
 
 # tars the objects to avoid 'content-length' - see WebDAVInterface.
 class WebDAVzipInterface(StorageInterface):
-    name = "webdavzip"
+    name = "webdav-zip"
 
     def __init__(self):
         super().__init__()
@@ -106,24 +106,6 @@ class WebDAVzipInterface(StorageInterface):
         object_name: str,
         where: str = "local",
     ) -> Tuple[bool, List[str]]:
-        if where == "local":
-            object_path = objects.object_path(
-                object_name=object_name,
-            )
-
-            return True, [
-                os.path.relpath(filename, start=object_path)
-                for filename in glob.glob(
-                    os.path.join(
-                        object_path,
-                        "**",
-                        "*",
-                    ),
-                    recursive=True,
-                )
-                if os.path.isfile(filename)
-            ]
-
         if where == "cloud":
             try:
                 if self.client.check(remote_path=f"{object_name}.zip"):
@@ -134,8 +116,10 @@ class WebDAVzipInterface(StorageInterface):
 
             return True, []
 
-        logger.error(f"Unknown 'where': {where}")
-        return False, []
+        return super().ls(
+            object_name=object_name,
+            where=where,
+        )
 
     def upload(
         self,

{bluer_objects-6.104.1 → bluer_objects-6.129.1}/bluer_objects/storage/__init__.py

@@ -2,6 +2,7 @@ from typing import Tuple, List
 
 from bluer_objects.storage.base import StorageInterface
 from bluer_objects.storage.WebDAV import WebDAVInterface
+from bluer_objects.storage.WebDAVrequest import WebDAVRequestInterface
 from bluer_objects.storage.WebDAVzip import WebDAVzipInterface
 from bluer_objects import env
 from bluer_objects.logger import logger
@@ -10,6 +11,8 @@ interface = StorageInterface()
 
 if env.BLUER_OBJECTS_STORAGE_INTERFACE == WebDAVInterface.name:
     interface = WebDAVInterface()
+elif env.BLUER_OBJECTS_STORAGE_INTERFACE == WebDAVRequestInterface.name:
+    interface = WebDAVRequestInterface()
 elif env.BLUER_OBJECTS_STORAGE_INTERFACE == WebDAVzipInterface.name:
     interface = WebDAVzipInterface()
 else:

{bluer_objects-6.104.1 → bluer_objects-6.129.1}/bluer_objects/storage/base.py

@@ -1,5 +1,8 @@
+import os
+import glob
 from typing import Tuple, List
 
+from bluer_objects import objects
 from bluer_objects.logger import logger
 
 
@@ -8,7 +11,7 @@ class StorageInterface:
         self,
         do_dryrun: bool = True,
     ) -> bool:
-        return False
+        return True
 
     def download(
         self,
@@ -32,7 +35,32 @@ class StorageInterface:
         object_name: str,
         where: str = "local",
     ) -> Tuple[bool, List[str]]:
-        return True, []
+        if where == "local":
+            object_path = objects.object_path(
+                object_name=object_name,
+            )
+
+            return True, sorted(
+                [
+                    os.path.relpath(filename, start=object_path)
+                    for filename in glob.glob(
+                        os.path.join(
+                            object_path,
+                            "**",
+                            "*",
+                        ),
+                        recursive=True,
+                    )
+                    if os.path.isfile(filename)
+                ]
+            )
+
+        if where == "cloud":
+            logger.error("not implemented")
+            return False, []
+
+        logger.error(f"Unknown 'where': {where}")
+        return False, []
 
     def upload(
         self,

{bluer_objects-6.104.1 → bluer_objects-6.129.1}/bluer_objects/tests/test_env.py

@@ -2,6 +2,7 @@ from bluer_ai.tests.test_env import test_bluer_ai_env
 
 from bluer_objects import env
 from bluer_objects.storage.WebDAV import WebDAVInterface
+from bluer_objects.storage.WebDAVrequest import WebDAVRequestInterface
 from bluer_objects.storage.WebDAVzip import WebDAVzipInterface
 
 
@@ -14,6 +15,7 @@ def test_bluer_objects_env():
 
     assert env.BLUER_OBJECTS_STORAGE_INTERFACE in [
         WebDAVInterface.name,
+        WebDAVRequestInterface.name,
         WebDAVzipInterface.name,
     ]
 

bluer_objects-6.129.1/bluer_objects/tests/test_file_load_save_text.py

@@ -0,0 +1,46 @@
+import pytest
+
+from bluer_options import string
+
+from bluer_objects import file, objects
+from bluer_objects.file.load import load_text
+from bluer_objects.file.save import save_text
+from bluer_objects.tests.test_objects import test_object
+
+
+@pytest.mark.parametrize(
+    [
+        "filename",
+    ],
+    [
+        ["test.json"],
+        ["test.yaml"],
+        ["test.yaml"],
+    ],
+)
+def test_file_load_save_text(
+    test_object,
+    filename: str,
+):
+    filename_input = objects.path_of(
+        object_name=test_object,
+        filename=filename,
+    )
+    success, text_input = load_text(filename_input)
+    assert success
+
+    filename_test = file.add_suffix(
+        filename_input,
+        string.random(),
+    )
+    assert save_text(
+        filename_test,
+        text_input,
+    )
+
+    success, text_output = load_text(filename_test)
+    assert success
+
+    assert len(text_input) == len(text_output)
+    for line_input, line_output in zip(text_input, text_output):
+        assert line_input == line_output