bluer-objects 6.104.1__py3-none-any.whl → 6.464.1__py3-none-any.whl

bluer_objects/pdf/convert/pdf.py

@@ -0,0 +1,37 @@
+from bluer_objects import file, objects
+from bluer_objects.env import abcli_path_git
+from bluer_objects.logger import logger
+
+
+def convert_pdf(
+    source_filename: str,
+    suffix: str,
+    object_name: str,
+    list_of_pdfs: list[str],
+) -> bool:
+    logger.info("🌠 pdf found!")
+    filename_pdf = file.add_extension(
+        objects.path_of(
+            filename="docs/{}".format(
+                (
+                    suffix.split(abcli_path_git, 1)[1]
+                    if abcli_path_git in suffix
+                    else suffix
+                ),
+            ),
+            object_name=object_name,
+        ),
+        "pdf",
+    )
+
+    if filename_pdf not in list_of_pdfs:
+        list_of_pdfs.append(filename_pdf)
+
+    if file.exists(filename_pdf):
+        logger.info(f"✅ {filename_pdf}")
+        return True
+
+    return file.copy(
+        source_filename,
+        filename_pdf,
+    )

bluer_objects/sample.env

@@ -1,3 +1,9 @@
+S3_STORAGE_ENDPOINT_URL=
+S3_STORAGE_AWS_ACCESS_KEY_ID=
+S3_STORAGE_AWS_SECRET_ACCESS_KEY=
+
+MLFLOW_DEPLOYMENT=local
+
 WEBDAV_HOSTNAME=
 WEBDAV_LOGIN=
 WEBDAV_PASSWORD=

bluer_objects/storage/WebDAV.py

@@ -4,6 +4,7 @@ from webdav3.client import Client
 from bluer_objects.storage.base import StorageInterface
 from bluer_objects import env, file, path
 from bluer_objects import objects
+from bluer_objects.storage.policies import DownloadPolicy
 from bluer_objects.logger import logger
 
 
@@ -56,6 +57,7 @@ class WebDAVInterface(StorageInterface):
         object_name: str,
         filename: str = "",
         log: bool = True,
+        policy: DownloadPolicy = DownloadPolicy.NONE,
     ) -> bool:
         local_path = objects.path_of(
             object_name=object_name,
@@ -80,21 +82,21 @@ class WebDAVInterface(StorageInterface):
             object_name=object_name,
             filename=filename,
             log=log,
+            policy=policy,
         )
 
-    def ls(
-        self,
-        object_name: str,
-        where: str = "local",
-    ) -> Tuple[bool, List[str]]:
-        assert False, "not implemented"
-
     def upload(
         self,
         object_name: str,
         filename: str = "",
+        public: bool = False,
+        zip: bool = False,
         log: bool = True,
     ) -> bool:
+        if public or zip:
+            logger.error("public/zip upload not supported.")
+            return False
+
         if filename:
             remote_dir = "/".join([object_name] + filename.split("/")[:-1])
             if not self.mkdir(
@@ -118,5 +120,7 @@ class WebDAVInterface(StorageInterface):
         return super().upload(
             object_name=object_name,
             filename=filename,
+            public=public,
+            zip=zip,
             log=log,
         )

bluer_objects/storage/WebDAVrequest.py

@@ -0,0 +1,360 @@
+import requests
+from requests.auth import HTTPBasicAuth
+import glob
+from typing import Tuple, List
+from xml.etree import ElementTree as ET
+from tqdm import tqdm
+
+from bluer_objects.storage.base import StorageInterface
+from bluer_objects import env, file, path
+from bluer_objects import objects
+from bluer_objects.storage.policies import DownloadPolicy
+from bluer_objects.logger import logger
+
+
+# https://chatgpt.com/c/6824cf43-6738-8005-8733-54b6a77f20ee
+class WebDAVRequestInterface(StorageInterface):
+    name = "webdav-request"
+
+    def clear(
+        self,
+        do_dryrun: bool = True,
+        log: bool = True,
+        public: bool = False,
+    ) -> bool:
+        if not super().clear(
+            do_dryrun=do_dryrun,
+            log=log,
+            public=public,
+        ):
+            return False
+
+        success, list_of_objects = self.list_raw(
+            suffix="",
+            recursive=False,
+        )
+        if not success:
+            return False
+
+        count: int = 0
+        for thing in tqdm(list_of_objects):
+            thing_name = thing.split(f"{env.WEBDAV_LOGIN}/", 1)[1]
+            if not thing_name.startswith("test"):
+                continue
+
+            if not thing_name.endswith("/"):
+                continue
+
+            if not self.delete(
+                object_name=thing_name.split("/", 1)[0],
+                do_dryrun=do_dryrun,
+                log=log,
+            ):
+                return False
+
+            count += 1
+
+        logger.info(f"deleted {count} object(s).")
+        return True
+
+    def delete(
+        self,
+        object_name: str,
+        do_dryrun: bool = True,
+        log: bool = True,
+    ) -> bool:
+        if log:
+            logger.info(
+                "{}.delete({}){}".format(
+                    self.__class__.__name__,
+                    object_name,
+                    " dryrun" if do_dryrun else "",
+                )
+            )
+        if do_dryrun:
+            return True
+
+        try:
+            response = requests.request(
+                method="DELETE",
+                url=f"{env.WEBDAV_HOSTNAME}/{object_name}/",
+                auth=HTTPBasicAuth(
+                    env.WEBDAV_LOGIN,
+                    env.WEBDAV_PASSWORD,
+                ),
+            )
+        except Exception as e:
+            logger.error(e)
+            return False
+
+        if response.status_code in [200, 204]:
+            return True
+
+        logger.error(f"failed to delete: {response.status_code} - {response.text}")
+        return False
+
+    def mkdir(
+        self,
+        path: str,
+        log: bool = True,
+    ) -> bool:
+        url = f"{env.WEBDAV_HOSTNAME}/"
+        for folder in path.split("/"):
+            url = f"{url}{folder}/"
+
+            try:
+                response = requests.request(
+                    "MKCOL",
+                    url,
+                    auth=HTTPBasicAuth(
+                        env.WEBDAV_LOGIN,
+                        env.WEBDAV_PASSWORD,
+                    ),
+                )
+            except Exception as e:
+                logger.error(e)
+                return False
+
+            if response.status_code == 405:  # Already exists
+                continue
+
+            if response.status_code == 201:  # Created
+                if log:
+                    logger.info(
+                        "{}.mkdir {}".format(
+                            self.__class__.__name__,
+                            url.split(env.WEBDAV_HOSTNAME, 1)[1],
+                        )
+                    )
+                continue
+
+            logger.error(f"failed to create: {response.status_code} - {response.text}")
+            return False
+
+        return True
+
+    def download(
+        self,
+        object_name: str,
+        filename: str = "",
+        log: bool = True,
+        policy: DownloadPolicy = DownloadPolicy.NONE,
+    ) -> bool:
+        if filename:
+            local_path = objects.path_of(
+                object_name=object_name,
+                filename=filename,
+                create=True,
+            )
+
+            if not path.create(file.path(local_path)):
+                return False
+
+            url = f"{env.WEBDAV_HOSTNAME}/{object_name}/{filename}"
+
+            try:
+                response = requests.get(
+                    url,
+                    auth=HTTPBasicAuth(
+                        env.WEBDAV_LOGIN,
+                        env.WEBDAV_PASSWORD,
+                    ),
+                )
+            except Exception as e:
+                logger.error(e)
+                return False
+
+            if response.status_code == 404:  # object not found
+                return True
+
+            if response.status_code == 200:
+                try:
+                    with open(local_path, "wb") as file_:
+                        file_.write(response.content)
+                except Exception as e:
+                    logger.error(e)
+                    return False
+
+                return super().download(
+                    object_name=object_name,
+                    filename=filename,
+                    log=log,
+                    policy=policy,
+                )
+
+            logger.error(f"failed to download: {response.status_code}")
+            return False
+
+        success, list_of_files = self.ls(
+            object_name=object_name,
+            where="cloud",
+        )
+        if not success:
+            return False
+
+        for filename_ in tqdm(list_of_files):
+            if not self.download(
+                object_name=object_name,
+                filename=filename_,
+                log=log,
+                policy=policy,
+            ):
+                return False
+
+        return True
+
+    def list_raw(
+        self,
+        suffix: str,
+        recursive: bool,
+    ) -> Tuple[bool, List[str]]:
+        # https://chatgpt.com/c/6824f8d3-d9c0-8005-a7fa-d646f812f4b7
+        headers = {
+            "Depth": "infinity" if recursive else "1",
+            "Content-Type": "application/xml",
+        }
+
+        # Minimal PROPFIND XML body
+        data = """<?xml version="1.0"?>
+        <d:propfind xmlns:d="DAV:">
+        <d:prop><d:displayname/></d:prop>
+        </d:propfind>"""
+
+        try:
+            response = requests.request(
+                method="PROPFIND",
+                url=f"{env.WEBDAV_HOSTNAME}/{suffix}",
+                data=data,
+                headers=headers,
+                auth=HTTPBasicAuth(
+                    env.WEBDAV_LOGIN,
+                    env.WEBDAV_PASSWORD,
+                ),
+            )
+        except Exception as e:
+            logger.error(e)
+            return False, []
+
+        if response.status_code == 404:  # object not found
+            return True, []
+
+        if response.status_code in (207, 207):
+            tree = ET.fromstring(response.content)
+            ns = {"d": "DAV:"}
+            list_of_files = []
+            for resp in tree.findall("d:response", ns):
+                href = resp.find("d:href", ns).text
+                list_of_files.append(href)
+
+            return True, list_of_files
+
+        logger.error(f"failed to list: {response.status_code} - {response.text}")
+        return False, []
+
+    def ls(
+        self,
+        object_name: str,
+        where: str = "local",
+    ) -> Tuple[bool, List[str]]:
+        if where == "cloud":
+            success, list_of_files = self.list_raw(
+                suffix=f"{object_name}/",
+                recursive=True,
+            )
+
+            return success, sorted(
+                [
+                    filename
+                    for filename in [
+                        filename.split(f"{env.WEBDAV_LOGIN}/{object_name}/", 1)[1]
+                        for filename in list_of_files
+                        if not filename.endswith("/")
+                    ]
+                    if filename
+                ]
+            )
+
+        return super().ls(
+            object_name=object_name,
+            where=where,
+        )
+
+    def upload(
+        self,
+        object_name: str,
+        filename: str = "",
+        public: bool = False,
+        zip: bool = False,
+        log: bool = True,
+    ) -> bool:
+        if public or zip:
+            logger.error("public/zip upload not supported.")
+            return False
+
+        if filename:
+            if not self.mkdir(
+                path="{}/{}".format(
+                    object_name,
+                    file.path(filename),
+                ),
+                log=log,
+            ):
+                return False
+
+            url = f"{env.WEBDAV_HOSTNAME}/{object_name}/{filename}"
+
+            local_path = objects.path_of(
+                object_name=object_name,
+                filename=filename,
+            )
+
+            try:
+                with open(local_path, "rb") as file_data:
+                    response = requests.put(
+                        url,
+                        data=file_data,
+                        auth=HTTPBasicAuth(
+                            env.WEBDAV_LOGIN,
+                            env.WEBDAV_PASSWORD,
+                        ),
+                    )
+            except Exception as e:
+                logger.error(e)
+                return False
+
+            if response.status_code in [200, 201, 204]:
+                return super().upload(
+                    object_name=object_name,
+                    filename=filename,
+                    public=public,
+                    zip=zip,
+                    log=log,
+                )
+
+            logger.error(f"failed to upload: {response.status_code} - {response.text}")
+            return False
+
+        object_path = "{}/".format(objects.object_path(object_name=object_name))
+        for filename_ in tqdm(
+            sorted(
+                glob.glob(
+                    objects.path_of(
+                        object_name=object_name,
+                        filename="**",
+                    ),
+                    recursive=True,
+                )
+            )
+        ):
+            if not file.exists(filename_):
+                continue
+
+            if not self.upload(
+                object_name=object_name,
+                filename=filename_.split(object_path, 1)[1],
+                public=public,
+                log=log,
+            ):
+                return False
+
+        return True

bluer_objects/storage/WebDAVzip.py

@@ -5,15 +5,16 @@ from webdav3.client import Client
 from tqdm import tqdm
 
 from bluer_objects.storage.base import StorageInterface
-from bluer_objects import env, file, path
+from bluer_objects import env
 from bluer_objects import objects
-from bluer_objects.host import zip, unzip
+from bluer_objects.host import unzip
+from bluer_objects.storage.policies import DownloadPolicy
 from bluer_objects.logger import logger
 
 
 # tars the objects to avoid 'content-length' - see WebDAVInterface.
 class WebDAVzipInterface(StorageInterface):
-    name = "webdavzip"
+    name = "webdav-zip"
 
     def __init__(self):
         super().__init__()
@@ -29,13 +30,15 @@ class WebDAVzipInterface(StorageInterface):
     def clear(
         self,
         do_dryrun: bool = True,
+        log: bool = True,
+        public: bool = False,
     ) -> bool:
-        logger.info(
-            "{}.clear({})".format(
-                self.__class__.__name__,
-                "dryrun" if do_dryrun else "",
-            )
-        )
+        if not super().clear(
+            do_dryrun=do_dryrun,
+            log=log,
+            public=public,
+        ):
+            return False
 
         count: int = 0
         for thing in tqdm(self.client.list()):
@@ -65,6 +68,7 @@ class WebDAVzipInterface(StorageInterface):
         object_name: str,
         filename: str = "",
         log: bool = True,
+        policy: DownloadPolicy = DownloadPolicy.NONE,
     ) -> bool:
         object_path = objects.object_path(
             object_name=object_name,
@@ -99,6 +103,7 @@ class WebDAVzipInterface(StorageInterface):
         return super().download(
             object_name=object_name,
             log=log,
+            policy=policy,
         )
 
     def ls(
@@ -106,24 +111,6 @@ class WebDAVzipInterface(StorageInterface):
         object_name: str,
         where: str = "local",
     ) -> Tuple[bool, List[str]]:
-        if where == "local":
-            object_path = objects.object_path(
-                object_name=object_name,
-            )
-
-            return True, [
-                os.path.relpath(filename, start=object_path)
-                for filename in glob.glob(
-                    os.path.join(
-                        object_path,
-                        "**",
-                        "*",
-                    ),
-                    recursive=True,
-                )
-                if os.path.isfile(filename)
-            ]
-
         if where == "cloud":
             try:
                 if self.client.check(remote_path=f"{object_name}.zip"):
@@ -134,15 +121,23 @@ class WebDAVzipInterface(StorageInterface):
 
             return True, []
 
-        logger.error(f"Unknown 'where': {where}")
-        return False, []
+        return super().ls(
+            object_name=object_name,
+            where=where,
+        )
 
     def upload(
         self,
         object_name: str,
         filename: str = "",
+        public: bool = False,
+        zip: bool = False,
         log: bool = True,
     ) -> bool:
+        if public or zip:
+            logger.error("public/zip upload not supported.")
+            return False
+
         object_path = objects.object_path(object_name=object_name)
 
         if not zip(
@@ -165,5 +160,7 @@ class WebDAVzipInterface(StorageInterface):
 
         return super().upload(
             object_name=object_name,
+            public=public,
+            zip=zip,
             log=log,
         )

bluer_objects/storage/__init__.py

@@ -1,15 +1,22 @@
 from typing import Tuple, List
 
+from bluer_objects.storage.s3 import S3Interface
 from bluer_objects.storage.base import StorageInterface
 from bluer_objects.storage.WebDAV import WebDAVInterface
+from bluer_objects.storage.WebDAVrequest import WebDAVRequestInterface
 from bluer_objects.storage.WebDAVzip import WebDAVzipInterface
+from bluer_objects.storage.policies import DownloadPolicy
 from bluer_objects import env
 from bluer_objects.logger import logger
 
 interface = StorageInterface()
 
-if env.BLUER_OBJECTS_STORAGE_INTERFACE == WebDAVInterface.name:
+if env.BLUER_OBJECTS_STORAGE_INTERFACE == S3Interface.name:
+    interface = S3Interface()
+elif env.BLUER_OBJECTS_STORAGE_INTERFACE == WebDAVInterface.name:
     interface = WebDAVInterface()
+elif env.BLUER_OBJECTS_STORAGE_INTERFACE == WebDAVRequestInterface.name:
+    interface = WebDAVRequestInterface()
 elif env.BLUER_OBJECTS_STORAGE_INTERFACE == WebDAVzipInterface.name:
     interface = WebDAVzipInterface()
 else:
@@ -19,9 +26,13 @@ else:
 
 def clear(
     do_dryrun: bool = True,
+    log: bool = True,
+    public: bool = False,
 ) -> bool:
     return interface.clear(
         do_dryrun=do_dryrun,
+        log=log,
+        public=public,
     )
 
 
@@ -29,11 +40,13 @@ def download(
     object_name: str,
     filename: str = "",
     log: bool = True,
+    policy: DownloadPolicy = DownloadPolicy.NONE,
 ) -> bool:
     return interface.download(
         object_name=object_name,
         filename=filename,
         log=log,
+        policy=policy,
     )
 
 
@@ -47,13 +60,27 @@ def ls(
     )
 
 
+def ls_objects(
+    prefix: str,
+    where: str = "local",
+) -> Tuple[bool, List[str]]:
+    return interface.ls_objects(
+        prefix=prefix,
+        where=where,
+    )
+
+
 def upload(
     object_name: str,
     filename: str = "",
+    public: bool = False,
+    zip: bool = False,
     log: bool = True,
 ) -> bool:
     return interface.upload(
         object_name=object_name,
         filename=filename,
+        public=public,
+        zip=zip,
         log=log,
     )