bluer-objects 6.104.1__py3-none-any.whl → 6.377.1__py3-none-any.whl

bluer_objects/storage/__init__.py

@@ -1,15 +1,22 @@
 from typing import Tuple, List
 
+from bluer_objects.storage.s3 import S3Interface
 from bluer_objects.storage.base import StorageInterface
 from bluer_objects.storage.WebDAV import WebDAVInterface
+from bluer_objects.storage.WebDAVrequest import WebDAVRequestInterface
 from bluer_objects.storage.WebDAVzip import WebDAVzipInterface
+from bluer_objects.storage.policies import DownloadPolicy
 from bluer_objects import env
 from bluer_objects.logger import logger
 
 interface = StorageInterface()
 
-if env.BLUER_OBJECTS_STORAGE_INTERFACE == WebDAVInterface.name:
+if env.BLUER_OBJECTS_STORAGE_INTERFACE == S3Interface.name:
+    interface = S3Interface()
+elif env.BLUER_OBJECTS_STORAGE_INTERFACE == WebDAVInterface.name:
     interface = WebDAVInterface()
+elif env.BLUER_OBJECTS_STORAGE_INTERFACE == WebDAVRequestInterface.name:
+    interface = WebDAVRequestInterface()
 elif env.BLUER_OBJECTS_STORAGE_INTERFACE == WebDAVzipInterface.name:
     interface = WebDAVzipInterface()
 else:
@@ -19,9 +26,13 @@ else:
 
 def clear(
     do_dryrun: bool = True,
+    log: bool = True,
+    public: bool = False,
 ) -> bool:
     return interface.clear(
         do_dryrun=do_dryrun,
+        log=log,
+        public=public,
     )
 
 
@@ -29,11 +40,13 @@ def download(
     object_name: str,
     filename: str = "",
     log: bool = True,
+    policy: DownloadPolicy = DownloadPolicy.NONE,
 ) -> bool:
     return interface.download(
         object_name=object_name,
         filename=filename,
         log=log,
+        policy=policy,
     )
 
 
@@ -47,13 +60,27 @@ def ls(
     )
 
 
+def ls_objects(
+    prefix: str,
+    where: str = "local",
+) -> Tuple[bool, List[str]]:
+    return interface.ls_objects(
+        prefix=prefix,
+        where=where,
+    )
+
+
 def upload(
     object_name: str,
     filename: str = "",
+    public: bool = False,
+    zip: bool = False,
     log: bool = True,
 ) -> bool:
     return interface.upload(
         object_name=object_name,
         filename=filename,
+        public=public,
+        zip=zip,
         log=log,
     )

bluer_objects/storage/__main__.py

@@ -2,9 +2,11 @@ import argparse
 
 from blueness import module
 from blueness.argparse.generic import sys_exit
+from bluer_options.logger.config import log_list
 
 from bluer_objects import NAME
 from bluer_objects import storage
+from bluer_objects.storage.policies import DownloadPolicy
 from bluer_objects.logger import logger
 
 NAME = module.name(__file__, NAME)
@@ -13,7 +15,7 @@ parser = argparse.ArgumentParser(NAME)
 parser.add_argument(
     "task",
     type=str,
-    help="clear | download | ls | upload",
+    help="clear | download | ls | ls_objects | upload",
 )
 parser.add_argument(
     "--object_name",
@@ -24,6 +26,18 @@ parser.add_argument(
     type=str,
     default="",
 )
+parser.add_argument(
+    "--public",
+    type=int,
+    default=0,
+    help="0 | 1",
+)
+parser.add_argument(
+    "--zip",
+    type=int,
+    default=0,
+    help="0 | 1",
+)
 parser.add_argument(
     "--where",
     type=str,
@@ -47,6 +61,17 @@ parser.add_argument(
     default=1,
     help="0 | 1",
 )
+parser.add_argument(
+    "--policy",
+    type=str,
+    default="none",
+    help=" | ".join(sorted([policy.name.lower() for policy in DownloadPolicy])),
+)
+parser.add_argument(
+    "--prefix",
+    type=str,
+    default="",
+)
 args = parser.parse_args()
 
 delim = " " if args.delim == "space" else args.delim
@@ -55,11 +80,13 @@ success = False
 if args.task == "clear":
     success = storage.clear(
         do_dryrun=args.do_dryrun == 1,
+        public=args.public == 1,
     )
 elif args.task == "download":
     success = storage.download(
         object_name=args.object_name,
         filename=args.filename,
+        policy=DownloadPolicy[args.policy.upper()],
     )
 elif args.task == "ls":
     success, list_of_files = storage.ls(
@@ -68,18 +95,25 @@ elif args.task == "ls":
     )
 
     if args.log:
-        logger.info(
-            "{:,} file(s).".format(len(list_of_files)),
-        )
-        for index, filename in enumerate(list_of_files):
-            logger.info(f"#{index+1: 4d} - {filename}")
+        log_list(logger, "", list_of_files, "file(s)", 999)
     else:
         print(delim.join(list_of_files))
+elif args.task == "ls_objects":
+    success, list_of_objects = storage.ls_objects(
+        prefix=args.prefix,
+        where=args.where,
+    )
 
+    if args.log:
+        log_list(logger, "", list_of_objects, "objects(s)", 999)
+    else:
+        print(delim.join(list_of_objects))
 elif args.task == "upload":
     success = storage.upload(
         object_name=args.object_name,
         filename=args.filename,
+        zip=args.zip == 1,
+        public=args.public == 1,
     )
 else:
     success = None

bluer_objects/storage/base.py

@@ -1,27 +1,50 @@
+import os
+import glob
 from typing import Tuple, List
 
+from bluer_objects import objects
+from bluer_objects import path
+from bluer_objects.storage.policies import DownloadPolicy
 from bluer_objects.logger import logger
+from bluer_objects.env import ABCLI_OBJECT_ROOT
 
 
 class StorageInterface:
     def clear(
         self,
         do_dryrun: bool = True,
+        log: bool = True,
+        public: bool = False,
     ) -> bool:
-        return False
+        logger.info(
+            "{}.clear({})".format(
+                self.__class__.__name__,
+                ",".join(
+                    (["dryrun"] if do_dryrun else []) + (["public"] if public else [])
+                ),
+            )
+        )
+
+        return True
 
     def download(
         self,
         object_name: str,
         filename: str = "",
         log: bool = True,
+        policy: DownloadPolicy = DownloadPolicy.NONE,
     ) -> bool:
         if log:
             logger.info(
-                "{}.download {}{}".format(
+                "{}.download {}{}{}".format(
                     self.__class__.__name__,
                     object_name,
                     f"/{filename}" if filename else "",
+                    (
+                        ""
+                        if policy == DownloadPolicy.NONE
+                        else " - policy:{}".format(policy.name.lower())
+                    ),
                 )
             )
 
@@ -32,20 +55,76 @@ class StorageInterface:
         object_name: str,
         where: str = "local",
     ) -> Tuple[bool, List[str]]:
-        return True, []
+        if where == "local":
+            object_path = objects.object_path(
+                object_name=object_name,
+            )
+
+            return True, sorted(
+                [
+                    os.path.relpath(filename, start=object_path)
+                    for filename in glob.glob(
+                        os.path.join(
+                            object_path,
+                            "**",
+                            "*",
+                        ),
+                        recursive=True,
+                    )
+                    if os.path.isfile(filename)
+                ]
+            )
+
+        if where == "cloud":
+            logger.error("not implemented")
+            return False, []
+
+        logger.error(f"Unknown 'where': {where}")
+        return False, []
+
+    def ls_objects(
+        self,
+        prefix: str,
+        where: str = "local",
+    ) -> Tuple[bool, List[str]]:
+        if where == "local":
+            return True, sorted(
+                [
+                    os.path.relpath(dirname, start=ABCLI_OBJECT_ROOT)
+                    for dirname in glob.glob(
+                        os.path.join(
+                            ABCLI_OBJECT_ROOT,
+                            "*",
+                        ),
+                        recursive=False,
+                    )
+                    if not os.path.isfile(dirname)
+                    and path.name(dirname).startswith(prefix)
+                ]
+            )
+
+        if where == "cloud":
+            logger.error("not implemented")
+            return False, []
+
+        logger.error(f"Unknown 'where': {where}")
+        return False, []
 
     def upload(
         self,
         object_name: str,
         filename: str = "",
+        public: bool = False,
+        zip: bool = False,
         log: bool = True,
     ) -> bool:
         if log:
             logger.info(
-                "{}.upload {}{}".format(
+                "{}.upload {}{}{}".format(
                     self.__class__.__name__,
                     object_name,
-                    f"/{filename}" if filename else "",
+                    ".tar.gz" if zip else f"/{filename}" if filename else "",
+                    " [public]" if public else "",
                 )
             )
 

bluer_objects/storage/policies.py

@@ -0,0 +1,7 @@
+from enum import Enum, auto
+
+
+class DownloadPolicy(Enum):
+    NONE = auto()
+    DOESNT_EXIST = auto()
+    DIFFERENT = auto()

bluer_objects/storage/s3.py

@@ -0,0 +1,367 @@
+import boto3
+import os
+from botocore.exceptions import ClientError
+import glob
+from typing import Tuple, List
+from xml.etree import ElementTree as ET
+from tqdm import tqdm
+from functools import reduce
+
+from bluer_objects.storage.base import StorageInterface
+from bluer_objects.env import ABCLI_OBJECT_ROOT
+from bluer_objects import env, file, path
+from bluer_objects import objects
+from bluer_objects.storage.policies import DownloadPolicy
+from bluer_objects.logger import logger
+
+
+# https://docs.arvancloud.ir/fa/developer-tools/sdk/object-storage/
+class S3Interface(StorageInterface):
+    name = "s3"
+
+    def clear(
+        self,
+        do_dryrun: bool = True,
+        log: bool = True,
+        public: bool = False,
+    ) -> bool:
+        if not super().clear(
+            do_dryrun=do_dryrun,
+            log=log,
+            public=public,
+        ):
+            return False
+
+        bucket_name = env.S3_PUBLIC_STORAGE_BUCKET if public else env.S3_STORAGE_BUCKET
+
+        success, list_of_objects = self.list_of_objects(
+            prefix="test",
+            bucket_name=bucket_name,
+        )
+        if not success:
+            return success
+        logger.info(f"{len(list_of_objects)} object(s) to delete.")
+
+        for object_name in tqdm(list_of_objects):
+            if not self.delete(
+                object_name=object_name,
+                do_dryrun=do_dryrun,
+                bucket_name=bucket_name,
+            ):
+                return False
+
+        return True
+
+    def delete(
+        self,
+        object_name: str,
+        do_dryrun: bool = True,
+        log: bool = True,
+        bucket_name: str = env.S3_STORAGE_BUCKET,
+    ) -> bool:
+        if log:
+            logger.info(
+                "{}.delete({}){}".format(
+                    self.__class__.__name__,
+                    object_name,
+                    " dryrun" if do_dryrun else "",
+                )
+            )
+        if do_dryrun:
+            return True
+
+        try:
+            s3 = boto3.resource(
+                "s3",
+                endpoint_url=env.S3_STORAGE_ENDPOINT_URL,
+                aws_access_key_id=env.S3_STORAGE_AWS_ACCESS_KEY_ID,
+                aws_secret_access_key=env.S3_STORAGE_AWS_SECRET_ACCESS_KEY,
+            )
+            bucket = s3.Bucket(bucket_name)
+
+            if object_name.endswith(".tar.gz"):
+                delete_requests = [{"Key": object_name}]
+            else:
+                objects_to_delete = bucket.objects.filter(Prefix=f"{object_name}/")
+                delete_requests = [{"Key": obj.key} for obj in objects_to_delete]
+
+            if not delete_requests:
+                logger.warning(f"no files found under {object_name}.")
+                return True
+
+            bucket.delete_objects(Delete={"Objects": delete_requests})
+        except Exception as e:
+            logger.error(e)
+            return False
+
+        return True
+
+    def download(
+        self,
+        object_name: str,
+        filename: str = "",
+        log: bool = True,
+        policy: DownloadPolicy = DownloadPolicy.NONE,
+    ) -> bool:
+        if filename:
+            local_path = objects.path_of(
+                object_name=object_name,
+                filename=filename,
+                create=True,
+            )
+
+            if policy == DownloadPolicy.DOESNT_EXIST and file.exists(local_path):
+                if log:
+                    logger.info(f"✅ {filename}")
+                return True
+
+            if not path.create(file.path(local_path)):
+                return False
+
+            try:
+                s3_resource = boto3.resource(
+                    "s3",
+                    endpoint_url=env.S3_STORAGE_ENDPOINT_URL,
+                    aws_access_key_id=env.S3_STORAGE_AWS_ACCESS_KEY_ID,
+                    aws_secret_access_key=env.S3_STORAGE_AWS_SECRET_ACCESS_KEY,
+                )
+            except Exception as e:
+                logger.error(e)
+                return False
+
+            try:
+                bucket = s3_resource.Bucket(env.S3_STORAGE_BUCKET)
+
+                bucket.download_file(
+                    f"{object_name}/{filename}",
+                    local_path,
+                )
+            except ClientError as e:
+                if int(e.response["Error"]["Code"]) == 404:  # Not found
+                    return True
+                logger.error(e)
+                return False
+            except Exception as e:
+                logger.error(e)
+                return False
+
+            return super().download(
+                object_name=object_name,
+                filename=filename,
+                log=log,
+                policy=policy,
+            )
+
+        success, list_of_files = self.ls(
+            object_name=object_name,
+            where="cloud",
+        )
+        if not success:
+            return False
+
+        for filename_ in tqdm(list_of_files):
+            if not self.download(
+                object_name=object_name,
+                filename=filename_,
+                log=log,
+                policy=policy,
+            ):
+                return False
+
+        return True
+
+    def list_of_objects(
+        self,
+        prefix: str = "",
+        bucket_name: str = env.S3_STORAGE_BUCKET,
+    ) -> Tuple[bool, List[str]]:
+        try:
+            s3 = boto3.client(
+                "s3",
+                endpoint_url=env.S3_STORAGE_ENDPOINT_URL,
+                aws_access_key_id=env.S3_STORAGE_AWS_ACCESS_KEY_ID,
+                aws_secret_access_key=env.S3_STORAGE_AWS_SECRET_ACCESS_KEY,
+            )
+
+            paginator = s3.get_paginator("list_objects_v2")
+            pages = paginator.paginate(
+                Bucket=bucket_name,
+                Prefix=prefix,
+            )
+        except Exception as e:
+            logger.error(e)
+            return False, []
+
+        list_of_objects = sorted(
+            list(
+                set(
+                    reduce(
+                        lambda x, y: x + y,
+                        [
+                            [
+                                obj["Key"].split("/", 1)[0]
+                                for obj in page.get("Contents", [])
+                            ]
+                            for page in pages
+                        ],
+                        [],
+                    )
+                )
+            )
+        )
+
+        return True, list_of_objects
+
+    def ls(
+        self,
+        object_name: str,
+        where: str = "local",
+    ) -> Tuple[bool, List[str]]:
+        if where == "cloud":
+            try:
+                s3 = boto3.client(
+                    "s3",
+                    endpoint_url=env.S3_STORAGE_ENDPOINT_URL,
+                    aws_access_key_id=env.S3_STORAGE_AWS_ACCESS_KEY_ID,
+                    aws_secret_access_key=env.S3_STORAGE_AWS_SECRET_ACCESS_KEY,
+                )
+
+                prefix = f"{object_name}/"
+
+                paginator = s3.get_paginator("list_objects_v2")
+                pages = paginator.paginate(
+                    Bucket=env.S3_STORAGE_BUCKET,
+                    Prefix=prefix,
+                )
+            except Exception as e:
+                logger.error(e)
+                return False, []
+
+            try:
+                list_of_files = sorted(
+                    reduce(
+                        lambda x, y: x + y,
+                        [
+                            [
+                                obj["Key"].split(prefix, 1)[1]
+                                for obj in page.get("Contents", [])
+                            ]
+                            for page in pages
+                        ],
+                        [],
+                    )
+                )
+            except Exception as e:
+                logger.error(e)
+                return False, []
+
+            return True, list_of_files
+
+        return super().ls(
+            object_name=object_name,
+            where=where,
+        )
+
+    def ls_objects(
+        self,
+        prefix: str,
+        where: str = "local",
+    ) -> Tuple[bool, List[str]]:
+        if where == "cloud":
+            return self.list_of_objects(prefix)
+
+        return super().ls_objects(
+            prefix=prefix,
+            where=where,
+        )
+
+    def upload(
+        self,
+        object_name: str,
+        filename: str = "",
+        public: bool = False,
+        zip: bool = False,
+        log: bool = True,
+    ) -> bool:
+        if filename or zip:
+            local_path = (
+                os.path.join(
+                    ABCLI_OBJECT_ROOT,
+                    f"{object_name}.tar.gz",
+                )
+                if zip
+                else objects.path_of(
+                    object_name=object_name,
+                    filename=filename,
+                )
+            )
+
+            bucket_name = (
+                env.S3_PUBLIC_STORAGE_BUCKET if public else env.S3_STORAGE_BUCKET
+            )
+
+            try:
+                s3_resource = boto3.resource(
+                    "s3",
+                    endpoint_url=env.S3_STORAGE_ENDPOINT_URL,
+                    aws_access_key_id=env.S3_STORAGE_AWS_ACCESS_KEY_ID,
+                    aws_secret_access_key=env.S3_STORAGE_AWS_SECRET_ACCESS_KEY,
+                )
+
+                bucket = s3_resource.Bucket(bucket_name)
+
+                with open(local_path, "rb") as fp:
+                    bucket.put_object(
+                        ACL="public-read" if public else "private",
+                        Body=fp,
+                        Key=(
+                            f"{object_name}.tar.gz"
+                            if zip
+                            else f"{object_name}/{filename}"
+                        ),
+                    )
+            except ClientError as e:
+                logger.error(e)
+                return False
+
+            if public:
+                logger.info(
+                    "🔗 https://{}.{}/{}".format(
+                        bucket_name,
+                        env.S3_STORAGE_ENDPOINT_URL.split("https://", 1)[1],
+                        f"{object_name}.tar.gz" if zip else f"{object_name}/{filename}",
+                    )
+                )
+
+            return super().upload(
+                object_name=object_name,
+                filename=filename,
+                public=public,
+                zip=zip,
+                log=log,
+            )
+
+        object_path = "{}/".format(objects.object_path(object_name=object_name))
+        for filename_ in tqdm(
+            sorted(
+                glob.glob(
+                    objects.path_of(
+                        object_name=object_name,
+                        filename="**",
+                    ),
+                    recursive=True,
+                )
+            )
+        ):
+            if not file.exists(filename_):
+                continue
+
+            if not self.upload(
+                object_name=object_name,
+                filename=filename_.split(object_path, 1)[1],
+                public=public,
+                log=log,
+            ):
+                return False
+
+        return True

bluer_objects/testing/__main__.py

@@ -19,12 +19,18 @@ parser.add_argument(
     "--object_name",
     type=str,
 )
+parser.add_argument(
+    "--depth",
+    type=int,
+    default=10,
+)
 args = parser.parse_args()
 
 success = False
 if args.task == "create_test_asset":
     success = create_test_asset(
         object_name=args.object_name,
+        depth=args.depth,
     )
 else:
     success = None