bluer-objects 6.139.1__tar.gz → 6.151.1__tar.gz

{bluer_objects-6.139.1/bluer_objects.egg-info → bluer_objects-6.151.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bluer_objects
-Version: 6.139.1
+Version: 6.151.1
 Summary: 🌀 Object management in Bash.
 Home-page: https://github.com/kamangir/bluer-objects
 Author: Arash Abadpour (Kamangir)
@@ -12,6 +12,7 @@ Classifier: Operating System :: OS Independent
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: bluer_ai
+Requires-Dist: boto3
 Requires-Dist: webdavclient3
 Requires-Dist: dill
 Requires-Dist: mlflow
@@ -63,6 +64,6 @@ pip install bluer-objects
 
 [![pylint](https://github.com/kamangir/bluer-objects/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/bluer-objects/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/bluer-objects/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/bluer-objects.svg)](https://pypi.org/project/bluer-objects/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/bluer-objects)](https://pypistats.org/packages/bluer-objects)
 
-built by 🌀 [`bluer README`](https://github.com/kamangir/bluer-objects/tree/main/bluer_objects/README), based on 🌀 [`bluer_objects-6.139.1`](https://github.com/kamangir/bluer-objects).
+built by 🌀 [`bluer README`](https://github.com/kamangir/bluer-objects/tree/main/bluer_objects/README), based on 🌀 [`bluer_objects-6.151.1`](https://github.com/kamangir/bluer-objects).
 
 built by 🌀 [`blueness-3.118.1`](https://github.com/kamangir/blueness).

{bluer_objects-6.139.1 → bluer_objects-6.151.1}/README.md

@@ -30,4 +30,4 @@ pip install bluer-objects
 
 [![pylint](https://github.com/kamangir/bluer-objects/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/bluer-objects/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/bluer-objects/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/bluer-objects.svg)](https://pypi.org/project/bluer-objects/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/bluer-objects)](https://pypistats.org/packages/bluer-objects)
 
-built by 🌀 [`bluer README`](https://github.com/kamangir/bluer-objects/tree/main/bluer_objects/README), based on 🌀 [`bluer_objects-6.139.1`](https://github.com/kamangir/bluer-objects).
+built by 🌀 [`bluer README`](https://github.com/kamangir/bluer-objects/tree/main/bluer_objects/README), based on 🌀 [`bluer_objects-6.151.1`](https://github.com/kamangir/bluer-objects).

{bluer_objects-6.139.1 → bluer_objects-6.151.1}/bluer_objects/__init__.py

@@ -4,7 +4,7 @@ ICON = "🌀"
 
 DESCRIPTION = f"{ICON} Object management in Bash."
 
-VERSION = "6.139.1"
+VERSION = "6.151.1"
 
 REPO_NAME = "bluer-objects"
 

bluer_objects-6.151.1/bluer_objects/config.env

@@ -0,0 +1,5 @@
+ABCLI_MLFLOW_EXPERIMENT_PREFIX=experiment/
+
+S3_STORAGE_BUCKET=kamangir
+
+BLUER_OBJECTS_STORAGE_INTERFACE=s3

{bluer_objects-6.139.1 → bluer_objects-6.151.1}/bluer_objects/env.py

@@ -35,6 +35,12 @@ DUMMY_TEXT = "This is some dummy text. This is some dummy text. This is some dum
 
 ABCLI_MLFLOW_EXPERIMENT_PREFIX = get_env("ABCLI_MLFLOW_EXPERIMENT_PREFIX")
 
+S3_STORAGE_BUCKET = get_env("S3_STORAGE_BUCKET")
+
+S3_STORAGE_ENDPOINT_URL = get_env("S3_STORAGE_ENDPOINT_URL")
+S3_STORAGE_AWS_ACCESS_KEY_ID = get_env("S3_STORAGE_AWS_ACCESS_KEY_ID")
+S3_STORAGE_AWS_SECRET_ACCESS_KEY = get_env("S3_STORAGE_AWS_SECRET_ACCESS_KEY")
+
 BLUER_OBJECTS_STORAGE_INTERFACE = get_env("BLUER_OBJECTS_STORAGE_INTERFACE")
 
 MLFLOW_DEPLOYMENT = get_env("MLFLOW_DEPLOYMENT", "local")

bluer_objects-6.151.1/bluer_objects/sample.env

@@ -0,0 +1,9 @@
+S3_STORAGE_ENDPOINT_URL=
+S3_STORAGE_AWS_ACCESS_KEY_ID=
+S3_STORAGE_AWS_SECRET_ACCESS_KEY=
+
+MLFLOW_DEPLOYMENT=local
+
+WEBDAV_HOSTNAME=
+WEBDAV_LOGIN=
+WEBDAV_PASSWORD=

{bluer_objects-6.139.1 → bluer_objects-6.151.1}/bluer_objects/storage/__init__.py

@@ -1,5 +1,6 @@
 from typing import Tuple, List
 
+from bluer_objects.storage.s3 import S3Interface
 from bluer_objects.storage.base import StorageInterface
 from bluer_objects.storage.WebDAV import WebDAVInterface
 from bluer_objects.storage.WebDAVrequest import WebDAVRequestInterface
@@ -9,7 +10,9 @@ from bluer_objects.logger import logger
 
 interface = StorageInterface()
 
-if env.BLUER_OBJECTS_STORAGE_INTERFACE == WebDAVInterface.name:
+if env.BLUER_OBJECTS_STORAGE_INTERFACE == S3Interface.name:
+    interface = S3Interface()
+elif env.BLUER_OBJECTS_STORAGE_INTERFACE == WebDAVInterface.name:
     interface = WebDAVInterface()
 elif env.BLUER_OBJECTS_STORAGE_INTERFACE == WebDAVRequestInterface.name:
     interface = WebDAVRequestInterface()

bluer_objects-6.151.1/bluer_objects/storage/s3.py

@@ -0,0 +1,186 @@
+import boto3
+from botocore.exceptions import ClientError
+import glob
+from typing import Tuple, List
+from xml.etree import ElementTree as ET
+from tqdm import tqdm
+from functools import reduce
+
+from bluer_objects.storage.base import StorageInterface
+from bluer_objects import env, file, path
+from bluer_objects import objects
+from bluer_objects.logger import logger
+
+
+# https://docs.arvancloud.ir/fa/developer-tools/sdk/object-storage/
+class S3Interface(StorageInterface):
+    name = "s3"
+
+    def download(
+        self,
+        object_name: str,
+        filename: str = "",
+        log: bool = True,
+    ) -> bool:
+        if filename:
+            local_path = objects.path_of(
+                object_name=object_name,
+                filename=filename,
+                create=True,
+            )
+
+            if not path.create(file.path(local_path)):
+                return False
+
+            try:
+                s3_resource = boto3.resource(
+                    "s3",
+                    endpoint_url=env.S3_STORAGE_ENDPOINT_URL,
+                    aws_access_key_id=env.S3_STORAGE_AWS_ACCESS_KEY_ID,
+                    aws_secret_access_key=env.S3_STORAGE_AWS_SECRET_ACCESS_KEY,
+                )
+            except Exception as e:
+                logger.error(e)
+                return False
+
+            try:
+                bucket = s3_resource.Bucket(env.S3_STORAGE_BUCKET)
+
+                bucket.download_file(
+                    f"{object_name}/{filename}",
+                    local_path,
+                )
+            except ClientError as e:
+                if int(e.response["Error"]["Code"]) == 404:  # Not found
+                    return True
+                logger.error(e)
+                return False
+
+            return super().download(
+                object_name=object_name,
+                filename=filename,
+                log=log,
+            )
+
+        success, list_of_files = self.ls(
+            object_name=object_name,
+            where="cloud",
+        )
+        if not success:
+            return False
+
+        for filename_ in tqdm(list_of_files):
+            if not self.download(
+                object_name=object_name,
+                filename=filename_,
+                log=log,
+            ):
+                return False
+
+        return True
+
+    def ls(
+        self,
+        object_name: str,
+        where: str = "local",
+    ) -> Tuple[bool, List[str]]:
+        if where == "cloud":
+            try:
+                s3 = boto3.client(
+                    "s3",
+                    endpoint_url=env.S3_STORAGE_ENDPOINT_URL,
+                    aws_access_key_id=env.S3_STORAGE_AWS_ACCESS_KEY_ID,
+                    aws_secret_access_key=env.S3_STORAGE_AWS_SECRET_ACCESS_KEY,
+                )
+
+                prefix = f"{object_name}/"
+
+                paginator = s3.get_paginator("list_objects_v2")
+                pages = paginator.paginate(
+                    Bucket=env.S3_STORAGE_BUCKET,
+                    Prefix=prefix,
+                )
+            except Exception as e:
+                logger.error(e)
+                return False, []
+
+            return True, sorted(
+                reduce(
+                    lambda x, y: x + y,
+                    [
+                        [
+                            obj["Key"].split(prefix, 1)[1]
+                            for obj in page.get("Contents", [])
+                        ]
+                        for page in pages
+                    ],
+                    [],
+                )
+            )
+
+        return super().ls(
+            object_name=object_name,
+            where=where,
+        )
+
+    def upload(
+        self,
+        object_name: str,
+        filename: str = "",
+        log: bool = True,
+    ) -> bool:
+        if filename:
+            local_path = objects.path_of(
+                object_name=object_name,
+                filename=filename,
+            )
+
+            try:
+                s3_resource = boto3.resource(
+                    "s3",
+                    endpoint_url=env.S3_STORAGE_ENDPOINT_URL,
+                    aws_access_key_id=env.S3_STORAGE_AWS_ACCESS_KEY_ID,
+                    aws_secret_access_key=env.S3_STORAGE_AWS_SECRET_ACCESS_KEY,
+                )
+
+                bucket = s3_resource.Bucket(env.S3_STORAGE_BUCKET)
+
+                with open(local_path, "rb") as fp:
+                    bucket.put_object(
+                        ACL="private",
+                        Body=fp,
+                        Key=f"{object_name}/{filename}",
+                    )
+            except ClientError as e:
+                logger.error(e)
+                return False
+
+            return super().upload(
+                object_name=object_name,
+                filename=filename,
+                log=log,
+            )
+
+        object_path = "{}/".format(objects.object_path(object_name=object_name))
+        for filename_ in tqdm(
+            sorted(
+                glob.glob(
+                    objects.path_of(
+                        object_name=object_name,
+                        filename="**",
+                    ),
+                    recursive=True,
+                )
+            )
+        ):
+            if not file.exists(filename_):
+                continue
+
+            if not self.upload(
+                object_name=object_name,
+                filename=filename_.split(object_path, 1)[1],
+                log=log,
+            ):
+                return False
+
+        return True

{bluer_objects-6.139.1 → bluer_objects-6.151.1}/bluer_objects/tests/test_env.py

@@ -1,9 +1,12 @@
 from bluer_ai.tests.test_env import test_bluer_ai_env
 
 from bluer_objects import env
-from bluer_objects.storage.WebDAV import WebDAVInterface
-from bluer_objects.storage.WebDAVrequest import WebDAVRequestInterface
-from bluer_objects.storage.WebDAVzip import WebDAVzipInterface
+from bluer_objects.storage import (
+    S3Interface,
+    WebDAVInterface,
+    WebDAVRequestInterface,
+    WebDAVzipInterface,
+)
 
 
 def test_required_env():
@@ -13,7 +16,14 @@ def test_required_env():
 def test_bluer_objects_env():
     assert env.ABCLI_MLFLOW_EXPERIMENT_PREFIX
 
+    assert env.S3_STORAGE_BUCKET
+
+    assert env.S3_STORAGE_ENDPOINT_URL
+    assert env.S3_STORAGE_AWS_ACCESS_KEY_ID
+    assert env.S3_STORAGE_AWS_SECRET_ACCESS_KEY
+
     assert env.BLUER_OBJECTS_STORAGE_INTERFACE in [
+        S3Interface.name,
         WebDAVInterface.name,
         WebDAVRequestInterface.name,
         WebDAVzipInterface.name,

{bluer_objects-6.139.1 → bluer_objects-6.151.1}/bluer_objects/tests/test_file_load_save.py

@@ -78,7 +78,6 @@ def test_file_load_save(
         )
 
 
-@pytest.mark.unit
 @pytest.mark.parametrize(
     ["size", "dtype"],
     [

bluer_objects-6.151.1/bluer_objects/tests/test_storage_s3.py

@@ -0,0 +1,67 @@
+from bluer_objects import objects
+from bluer_objects.testing import create_test_asset
+from bluer_objects.storage import S3Interface
+
+
+def test_storage_s3():
+    object_name = objects.unique_object("test_storage_s3")
+
+    assert create_test_asset(
+        object_name=object_name,
+        depth=10,
+    )
+
+    storage = S3Interface()
+
+    success, list_of_files_local = storage.ls(
+        object_name=object_name,
+        where="local",
+    )
+    assert success
+    assert list_of_files_local
+
+    success, list_of_files_cloud = storage.ls(
+        object_name=object_name,
+        where="cloud",
+    )
+    assert success
+    assert not list_of_files_cloud
+
+    for filename in [
+        "this.yaml",
+        "subfolder/this.yaml",
+        "test-00.png",
+    ]:
+        assert storage.upload(
+            object_name=object_name,
+            filename=filename,
+        )
+
+    success, list_of_files_cloud = storage.ls(
+        object_name=object_name,
+        where="cloud",
+    )
+    assert success
+    assert list_of_files_cloud
+
+    assert storage.upload(object_name=object_name)
+
+    success, list_of_files_cloud = storage.ls(
+        object_name=object_name,
+        where="cloud",
+    )
+    assert success
+    assert list_of_files_cloud
+    assert list_of_files_cloud == list_of_files_local
+
+    for filename in [
+        "this.yaml",
+        "subfolder/this.yaml",
+        "test-00.png",
+    ]:
+        assert storage.download(
+            object_name=object_name,
+            filename=filename,
+        )
+
+    assert storage.download(object_name=object_name)

{bluer_objects-6.139.1 → bluer_objects-6.151.1}/bluer_objects/tests/test_storage_webdav_request.py

@@ -1,8 +1,11 @@
+import pytest
+
 from bluer_objects import objects
 from bluer_objects.testing import create_test_asset
 from bluer_objects.storage import WebDAVRequestInterface
 
 
+@pytest.mark.skip(reason="nodisk is super slow")
 def test_storage_webdav_request():
     object_name = objects.unique_object("test_storage_webdav_request")
 

{bluer_objects-6.139.1 → bluer_objects-6.151.1}/bluer_objects/tests/test_storage_webdav_zip.py

@@ -1,8 +1,11 @@
+import pytest
+
 from bluer_objects import objects
 from bluer_objects.testing import create_test_asset
 from bluer_objects.storage import WebDAVzipInterface
 
 
+@pytest.mark.skip(reason="nodisk is super slow")
 def test_storage_webdav_zip():
     object_name = objects.unique_object("test_storage_webdav_zip")
 

{bluer_objects-6.139.1 → bluer_objects-6.151.1/bluer_objects.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bluer_objects
-Version: 6.139.1
+Version: 6.151.1
 Summary: 🌀 Object management in Bash.
 Home-page: https://github.com/kamangir/bluer-objects
 Author: Arash Abadpour (Kamangir)
@@ -12,6 +12,7 @@ Classifier: Operating System :: OS Independent
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: bluer_ai
+Requires-Dist: boto3
 Requires-Dist: webdavclient3
 Requires-Dist: dill
 Requires-Dist: mlflow
@@ -63,6 +64,6 @@ pip install bluer-objects
 
 [![pylint](https://github.com/kamangir/bluer-objects/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/bluer-objects/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/bluer-objects/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/bluer-objects.svg)](https://pypi.org/project/bluer-objects/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/bluer-objects)](https://pypistats.org/packages/bluer-objects)
 
-built by 🌀 [`bluer README`](https://github.com/kamangir/bluer-objects/tree/main/bluer_objects/README), based on 🌀 [`bluer_objects-6.139.1`](https://github.com/kamangir/bluer-objects).
+built by 🌀 [`bluer README`](https://github.com/kamangir/bluer-objects/tree/main/bluer_objects/README), based on 🌀 [`bluer_objects-6.151.1`](https://github.com/kamangir/bluer-objects).
 
 built by 🌀 [`blueness-3.118.1`](https://github.com/kamangir/blueness).

{bluer_objects-6.139.1 → bluer_objects-6.151.1}/bluer_objects.egg-info/SOURCES.txt

@@ -127,6 +127,7 @@ bluer_objects/storage/WebDAVzip.py
 bluer_objects/storage/__init__.py
 bluer_objects/storage/__main__.py
 bluer_objects/storage/base.py
+bluer_objects/storage/s3.py
 bluer_objects/testing/__init__.py
 bluer_objects/testing/__main__.py
 bluer_objects/testing/functions.py
@@ -152,6 +153,7 @@ bluer_objects/tests/test_objects.py
 bluer_objects/tests/test_path.py
 bluer_objects/tests/test_storage.py
 bluer_objects/tests/test_storage_base.py
+bluer_objects/tests/test_storage_s3.py
 bluer_objects/tests/test_storage_webdav_request.py
 bluer_objects/tests/test_storage_webdav_zip.py
 bluer_objects/tests/test_testing.py

{bluer_objects-6.139.1 → bluer_objects-6.151.1}/bluer_objects.egg-info/requires.txt

@@ -1,4 +1,5 @@
 bluer_ai
+boto3
 webdavclient3
 dill
 mlflow

{bluer_objects-6.139.1 → bluer_objects-6.151.1}/requirements.txt

@@ -1,5 +1,6 @@
 bluer_ai
 
+boto3
 webdavclient3
 dill
 mlflow

bluer_objects-6.139.1/bluer_objects/config.env

@@ -1,3 +0,0 @@
-ABCLI_MLFLOW_EXPERIMENT_PREFIX=experiment/
-
-BLUER_OBJECTS_STORAGE_INTERFACE=webdav-request

bluer_objects-6.139.1/bluer_objects/sample.env

@@ -1,5 +0,0 @@
-MLFLOW_DEPLOYMENT=local
-
-WEBDAV_HOSTNAME=
-WEBDAV_LOGIN=
-WEBDAV_PASSWORD=