bluer-objects 6.24.1__py3-none-any.whl → 6.26.1__py3-none-any.whl

bluer_objects/storage/WebDAVzip.py

@@ -0,0 +1,84 @@
+from webdav3.client import Client
+
+from bluer_objects.storage.base import StorageInterface
+from bluer_objects import env, file, path
+from bluer_objects import objects
+from bluer_objects.host import zip, unzip
+from bluer_objects.logger import logger
+
+
+# tars the objects to avoid 'content-length' - see WebDAVInterface.
+class WebDAVzipInterface(StorageInterface):
+    name = "webdavzip"
+
+    def __init__(self):
+        super().__init__()
+
+        config = {
+            "webdav_hostname": env.WEBDAV_HOSTNAME,
+            "webdav_login": env.WEBDAV_LOGIN,
+            "webdav_password": env.WEBDAV_PASSWORD,
+        }
+
+        self.client = Client(config)
+
+    def download(
+        self,
+        object_name: str,
+        filename: str = "",
+        log: bool = True,
+    ) -> bool:
+        object_path = objects.object_path(object_name=object_name)
+        zip_filename = f"{object_path}.zip"
+
+        try:
+            self.client.download_sync(
+                remote_path=f"{object_name}.zip",
+                local_path=zip_filename,
+            )
+        except Exception as e:
+            logger.error(e)
+            return False
+
+        if not unzip(
+            zip_filename=zip_filename,
+            output_folder=object_path,
+            log=log,
+        ):
+            return False
+
+        return super().download(
+            object_name=object_name,
+            log=log,
+        )
+
+    def upload(
+        self,
+        object_name: str,
+        filename: str = "",
+        log: bool = True,
+    ) -> bool:
+        object_path = objects.object_path(object_name=object_name)
+
+        if not zip(
+            zip_filename=f"../{object_name}.zip",
+            input_folder=".",
+            work_dir=object_path,
+            log=log,
+        ):
+            return False
+
+        zip_filename = f"{object_path}.zip"
+        try:
+            self.client.upload_sync(
+                remote_path=f"{object_name}.zip",
+                local_path=zip_filename,
+            )
+        except Exception as e:
+            logger.error(e)
+            return False
+
+        return super().upload(
+            object_name=object_name,
+            log=log,
+        )

bluer_objects/storage/__init__.py

@@ -0,0 +1,39 @@
+from bluer_objects.storage.base import StorageInterface
+from bluer_objects.storage.WebDAV import WebDAVInterface
+from bluer_objects.storage.WebDAVzip import WebDAVzipInterface
+from bluer_objects import env
+from bluer_objects.logger import logger
+
+interface = StorageInterface()
+
+if env.BLUER_OBJECTS_STORAGE_INTERFACE == WebDAVInterface.name:
+    interface = WebDAVInterface()
+elif env.BLUER_OBJECTS_STORAGE_INTERFACE == WebDAVzipInterface.name:
+    interface = WebDAVzipInterface()
+else:
+    logger.error(f"{env.BLUER_OBJECTS_STORAGE_INTERFACE}: interface not found.")
+    assert False
+
+
+def download(
+    object_name: str,
+    filename: str = "",
+    log: bool = True,
+) -> bool:
+    return interface.download(
+        object_name=object_name,
+        filename=filename,
+        log=log,
+    )
+
+
+def upload(
+    object_name: str,
+    filename: str = "",
+    log: bool = True,
+) -> bool:
+    return interface.upload(
+        object_name=object_name,
+        filename=filename,
+        log=log,
+    )

bluer_objects/storage/__main__.py

@@ -0,0 +1,44 @@
+import argparse
+
+from blueness import module
+from blueness.argparse.generic import sys_exit
+
+from bluer_objects import NAME
+from bluer_objects import storage
+from bluer_objects.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+parser = argparse.ArgumentParser(NAME)
+parser.add_argument(
+    "task",
+    type=str,
+    help="download | upload",
+)
+parser.add_argument(
+    "--object_name",
+    type=str,
+)
+parser.add_argument(
+    "--filename",
+    type=str,
+    default="",
+)
+args = parser.parse_args()
+
+
+success = False
+if args.task == "download":
+    success = storage.download(
+        object_name=args.object_name,
+        filename=args.filename,
+    )
+elif args.task == "upload":
+    success = storage.upload(
+        object_name=args.object_name,
+        filename=args.filename,
+    )
+else:
+    success = None
+
+sys_exit(logger, NAME, args.task, success)

bluer_objects/storage/base.py

@@ -0,0 +1,37 @@
+from bluer_objects.logger import logger
+
+
+class StorageInterface:
+    def download(
+        self,
+        object_name: str,
+        filename: str = "",
+        log: bool = True,
+    ) -> bool:
+        if log:
+            logger.info(
+                "{}.download {}{}".format(
+                    self.__class__.__name__,
+                    object_name,
+                    f"/{filename}" if filename else "",
+                )
+            )
+
+        return True
+
+    def upload(
+        self,
+        object_name: str,
+        filename: str = "",
+        log: bool = True,
+    ) -> bool:
+        if log:
+            logger.info(
+                "{}.upload {}{}".format(
+                    self.__class__.__name__,
+                    object_name,
+                    f"/{filename}" if filename else "",
+                )
+            )
+
+        return True

bluer_objects/tests/test_download.py

@@ -0,0 +1,52 @@
+import pytest
+
+
+from bluer_objects import file, objects
+from bluer_objects import storage
+
+
+@pytest.mark.parametrize(
+    ["object_name", "filename"],
+    [[VANWATCH_TEST_OBJECT, "vancouver.geojson"]],
+)
+def test_objects_download(
+    object_name: str,
+    filename: str,
+):
+    filename_fullpath = objects.path_of(
+        filename=filename,
+        object_name=object_name,
+    )
+    if file.exists(filename_fullpath):
+        assert file.delete(filename_fullpath)
+
+    assert not file.exists(filename_fullpath)
+
+    assert objects.storage(object_name=object_name)
+
+    assert file.exists(filename_fullpath)
+
+
+@pytest.mark.parametrize(
+    ["object_name", "filename"],
+    [[VANWATCH_TEST_OBJECT, "vancouver.geojson"]],
+)
+def test_objects_download_filename(
+    object_name: str,
+    filename: str,
+):
+    filename_fullpath = objects.path_of(
+        filename=filename,
+        object_name=object_name,
+    )
+    if file.exists(filename_fullpath):
+        assert file.delete(filename_fullpath)
+
+    assert not file.exists(filename_fullpath)
+
+    assert storage.download(
+        object_name=object_name,
+        filename=filename,
+    )
+
+    assert file.exists(filename_fullpath)

bluer_objects/tests/test_env.py

@@ -1,6 +1,8 @@
 from bluer_ai.tests.test_env import test_bluer_ai_env
 
 from bluer_objects import env
+from bluer_objects.storage.WebDAV import WebDAVInterface
+from bluer_objects.storage.WebDAVzip import WebDAVzipInterface
 
 
 def test_required_env():
@@ -15,3 +17,12 @@ def test_bluer_objects_env():
     assert env.DATABRICKS_TOKEN
 
     assert env.ABCLI_MLFLOW_EXPERIMENT_PREFIX
+
+    assert env.BLUER_OBJECTS_STORAGE_INTERFACE in [
+        WebDAVInterface.name,
+        WebDAVzipInterface.name,
+    ]
+
+    assert env.WEBDAV_HOSTNAME
+    assert env.WEBDAV_LOGIN
+    assert env.WEBDAV_PASSWORD

bluer_objects/tests/test_graphics.py

@@ -3,6 +3,7 @@ import pytest
 from blueness import module
 
 from bluer_objects import file, objects, NAME
+from bluer_objects import storage
 from bluer_objects.env import VANWATCH_TEST_OBJECT
 from bluer_objects.logger import logger
 
@@ -13,7 +14,7 @@ NAME = module.name(__file__, NAME)
 def test_image():
     object_name = VANWATCH_TEST_OBJECT
 
-    assert objects.download(object_name)
+    assert storage.download(object_name)
 
     success, matrix = file.load_image(
         objects.path_of(

bluer_objects/tests/test_graphics_gif.py

@@ -2,6 +2,7 @@ import pytest
 import glob
 
 from bluer_objects import objects
+from bluer_objects import storage
 from bluer_objects.graphics.gif import generate_animated_gif
 
 
@@ -16,7 +17,7 @@ def test_graphics_gif_generate_animated_gif(
     object_name: str,
     scale: int,
 ):
-    assert objects.download(object_name)
+    assert storage.download(object_name)
 
     list_of_images = list(glob.glob(objects.path_of("*.png", object_name)))
     if object_name != "void":

bluer_objects/tests/test_objects.py

@@ -3,6 +3,7 @@ import pytest
 from blueness import module
 
 from bluer_objects import file, path, objects, NAME
+from bluer_objects import storage
 from bluer_objects.env import VANWATCH_TEST_OBJECT
 from bluer_objects.logger import logger
 
@@ -13,60 +14,13 @@ NAME = module.name(__file__, NAME)
 def test_object():
     object_name = VANWATCH_TEST_OBJECT
 
-    assert objects.download(object_name=object_name)
+    assert storage.download(object_name=object_name)
 
     yield object_name
 
     logger.info(f"deleting {NAME}.test_object ...")
 
 
-@pytest.mark.parametrize(
-    ["object_name", "filename"],
-    [[VANWATCH_TEST_OBJECT, "vancouver.geojson"]],
-)
-def test_objects_download(
-    object_name: str,
-    filename: str,
-):
-    filename_fullpath = objects.path_of(
-        filename=filename,
-        object_name=object_name,
-    )
-    if file.exists(filename_fullpath):
-        assert file.delete(filename_fullpath)
-
-    assert not file.exists(filename_fullpath)
-
-    assert objects.download(object_name=object_name)
-
-    assert file.exists(filename_fullpath)
-
-
-@pytest.mark.parametrize(
-    ["object_name", "filename"],
-    [[VANWATCH_TEST_OBJECT, "vancouver.geojson"]],
-)
-def test_objects_download_filename(
-    object_name: str,
-    filename: str,
-):
-    filename_fullpath = objects.path_of(
-        filename=filename,
-        object_name=object_name,
-    )
-    if file.exists(filename_fullpath):
-        assert file.delete(filename_fullpath)
-
-    assert not file.exists(filename_fullpath)
-
-    assert objects.download(
-        object_name=object_name,
-        filename=filename,
-    )
-
-    assert file.exists(filename_fullpath)
-
-
 @pytest.mark.parametrize(
     ["cloud"],
     [[True], [False]],
@@ -110,71 +64,3 @@ def test_objects_unique_object(prefix: str):
     object_name = objects.unique_object(prefix)
     assert object_name
     assert object_name.startswith(prefix)
-
-
-@pytest.mark.parametrize(
-    ["filename"],
-    [["vancouver.geojson"]],
-)
-def test_objects_upload(
-    filename: str,
-):
-    assert objects.download(object_name=VANWATCH_TEST_OBJECT)
-
-    object_name = objects.unique_object("test_objects_upload_filename")
-
-    source_filename = objects.path_of(
-        filename=filename,
-        object_name=VANWATCH_TEST_OBJECT,
-    )
-    destination_filename = objects.path_of(
-        filename=filename,
-        object_name=object_name,
-    )
-    assert file.copy(source_filename, destination_filename)
-
-    assert objects.upload(object_name=object_name)
-
-    assert file.delete(destination_filename)
-    assert not file.exists(destination_filename)
-
-    assert objects.download(object_name=object_name)
-
-    assert file.exists(destination_filename)
-
-
-@pytest.mark.parametrize(
-    ["filename"],
-    [["vancouver.geojson"]],
-)
-def test_objects_upload_filename(
-    filename: str,
-):
-    assert objects.download(object_name=VANWATCH_TEST_OBJECT)
-
-    object_name = objects.unique_object("test_objects_upload_filename")
-
-    source_filename = objects.path_of(
-        filename=filename,
-        object_name=VANWATCH_TEST_OBJECT,
-    )
-    destination_filename = objects.path_of(
-        filename=filename,
-        object_name=object_name,
-    )
-    assert file.copy(source_filename, destination_filename)
-
-    assert objects.upload(
-        object_name=object_name,
-        filename=filename,
-    )
-
-    assert file.delete(destination_filename)
-    assert not file.exists(destination_filename)
-
-    assert objects.download(
-        object_name=object_name,
-        filename=filename,
-    )
-
-    assert file.exists(destination_filename)

bluer_objects/tests/test_upload.py

@@ -0,0 +1,76 @@
+import pytest
+
+from blueness import module
+
+from bluer_objects import file, objects, NAME
+from bluer_objects import storage
+
+NAME = module.name(__file__, NAME)
+
+
+@pytest.mark.parametrize(
+    ["filename"],
+    [["vancouver.geojson"]],
+)
+def test_objects_upload(
+    filename: str,
+):
+    assert storage.download(object_name=VANWATCH_TEST_OBJECT)
+
+    object_name = objects.unique_object("test_objects_upload_filename")
+
+    source_filename = objects.path_of(
+        filename=filename,
+        object_name=VANWATCH_TEST_OBJECT,
+    )
+    destination_filename = objects.path_of(
+        filename=filename,
+        object_name=object_name,
+    )
+    assert file.copy(source_filename, destination_filename)
+
+    assert objects.upload(object_name=object_name)
+
+    assert file.delete(destination_filename)
+    assert not file.exists(destination_filename)
+
+    assert storage.download(object_name=object_name)
+
+    assert file.exists(destination_filename)
+
+
+@pytest.mark.parametrize(
+    ["filename"],
+    [["vancouver.geojson"]],
+)
+def test_objects_upload_filename(
+    filename: str,
+):
+    assert storage.download(object_name=VANWATCH_TEST_OBJECT)
+
+    object_name = objects.unique_object("test_objects_upload_filename")
+
+    source_filename = objects.path_of(
+        filename=filename,
+        object_name=VANWATCH_TEST_OBJECT,
+    )
+    destination_filename = objects.path_of(
+        filename=filename,
+        object_name=object_name,
+    )
+    assert file.copy(source_filename, destination_filename)
+
+    assert objects.upload(
+        object_name=object_name,
+        filename=filename,
+    )
+
+    assert file.delete(destination_filename)
+    assert not file.exists(destination_filename)
+
+    assert storage.download(
+        object_name=object_name,
+        filename=filename,
+    )
+
+    assert file.exists(destination_filename)

{bluer_objects-6.24.1.dist-info → bluer_objects-6.26.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bluer_objects
-Version: 6.24.1
+Version: 6.26.1
 Summary: 🌀 data objects for Bash.
 Home-page: https://github.com/kamangir/bluer-objects
 Author: Arash Abadpour (Kamangir)
@@ -13,6 +13,7 @@ Classifier: Operating System :: OS Independent
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: bluer_ai
+Requires-Dist: webdavclient3
 Requires-Dist: dill
 Requires-Dist: mlflow
 Requires-Dist: pandas
@@ -33,7 +34,7 @@ Dynamic: summary
 
 # 🌀 bluer-objects
 
-🌀 `bluer-objects` are the inputs and outputs of [AI algo](https://github.com/kamangir/giza). They are maintained in cloud storage and their metadata is tracked by [MLflow](https://mlflow.org/).
+🌀 `bluer-objects` are the inputs and outputs of [AI algo](https://github.com/kamangir/giza). They are maintained in cloud storage (supports [WebDav](https://pypi.org/project/webdavclient3/)) and their metadata is tracked by [MLflow](https://mlflow.org/).
 
 For example, the Sentinel-2 [datacube](https://github.com/kamangir/blue-geo/tree/main/blue_geo/datacube) `datacube-EarthSearch-sentinel_2_l1c-S2A_10UDC_20240731_0_L1C` and 🌐 [`@geo watch` outputs](https://github.com/kamangir/blue-geo/tree/main/blue_geo/watch) are `bluer-objects`.
 
@@ -45,13 +46,13 @@ Also home to [blue README](https://github.com/kamangir/bluer-objects/blob/main/b
 
 ---
 
-> 🌀 [`blue-objects`](https://github.com/kamangir/blue-objects) for the [Global South](https://github.com/kamangir/blue-south).
+> 🌀 [`blue-objects`](https://github.com/kamangir/blue-objects) for the [Global South](https://github.com/kamangir/bluer-south).
 
 ---
 
 
 [![pylint](https://github.com/kamangir/bluer-objects/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/bluer-objects/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/bluer-objects/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/bluer-objects.svg)](https://pypi.org/project/bluer-objects/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/bluer-objects)](https://pypistats.org/packages/bluer-objects)
 
-built by 🌀 [`bluer_options-5.21.1`](https://github.com/kamangir/awesome-bash-cli), based on 🌀 [`bluer_objects-6.24.1`](https://github.com/kamangir/bluer-objects).
+built by 🌀 [`bluer_options-5.26.1`](https://github.com/kamangir/awesome-bash-cli), based on 🌀 [`bluer_objects-6.26.1`](https://github.com/kamangir/bluer-objects).
 
 built by 🌀 [`blueness-3.96.1`](https://github.com/kamangir/blueness).