bluer-objects 6.25.1__tar.gz → 6.27.1__tar.gz

{bluer_objects-6.25.1/bluer_objects.egg-info → bluer_objects-6.27.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bluer_objects
-Version: 6.25.1
+Version: 6.27.1
 Summary: 🌀 data objects for Bash.
 Home-page: https://github.com/kamangir/bluer-objects
 Author: Arash Abadpour (Kamangir)
@@ -13,6 +13,7 @@ Classifier: Operating System :: OS Independent
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: bluer_ai
+Requires-Dist: webdavclient3
 Requires-Dist: dill
 Requires-Dist: mlflow
 Requires-Dist: pandas
@@ -33,7 +34,7 @@ Dynamic: summary
 
 # 🌀 bluer-objects
 
-🌀 `bluer-objects` are the inputs and outputs of [AI algo](https://github.com/kamangir/giza). They are maintained in cloud storage and their metadata is tracked by [MLflow](https://mlflow.org/).
+🌀 `bluer-objects` are the inputs and outputs of [AI algo](https://github.com/kamangir/giza). They are maintained in cloud storage (supports [WebDav](https://pypi.org/project/webdavclient3/)) and their metadata is tracked by [MLflow](https://mlflow.org/).
 
 For example, the Sentinel-2 [datacube](https://github.com/kamangir/blue-geo/tree/main/blue_geo/datacube) `datacube-EarthSearch-sentinel_2_l1c-S2A_10UDC_20240731_0_L1C` and 🌐 [`@geo watch` outputs](https://github.com/kamangir/blue-geo/tree/main/blue_geo/watch) are `bluer-objects`.
 
@@ -52,6 +53,6 @@ Also home to [blue README](https://github.com/kamangir/bluer-objects/blob/main/b
 
 [![pylint](https://github.com/kamangir/bluer-objects/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/bluer-objects/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/bluer-objects/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/bluer-objects.svg)](https://pypi.org/project/bluer-objects/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/bluer-objects)](https://pypistats.org/packages/bluer-objects)
 
-built by 🌀 [`bluer_options-5.24.1`](https://github.com/kamangir/awesome-bash-cli), based on 🌀 [`bluer_objects-6.25.1`](https://github.com/kamangir/bluer-objects).
+built by 🌀 [`bluer_options-5.26.1`](https://github.com/kamangir/awesome-bash-cli), based on 🌀 [`bluer_objects-6.27.1`](https://github.com/kamangir/bluer-objects).
 
 built by 🌀 [`blueness-3.96.1`](https://github.com/kamangir/blueness).

{bluer_objects-6.25.1 → bluer_objects-6.27.1}/README.md

@@ -1,6 +1,6 @@
 # 🌀 bluer-objects
 
-🌀 `bluer-objects` are the inputs and outputs of [AI algo](https://github.com/kamangir/giza). They are maintained in cloud storage and their metadata is tracked by [MLflow](https://mlflow.org/).
+🌀 `bluer-objects` are the inputs and outputs of [AI algo](https://github.com/kamangir/giza). They are maintained in cloud storage (supports [WebDav](https://pypi.org/project/webdavclient3/)) and their metadata is tracked by [MLflow](https://mlflow.org/).
 
 For example, the Sentinel-2 [datacube](https://github.com/kamangir/blue-geo/tree/main/blue_geo/datacube) `datacube-EarthSearch-sentinel_2_l1c-S2A_10UDC_20240731_0_L1C` and 🌐 [`@geo watch` outputs](https://github.com/kamangir/blue-geo/tree/main/blue_geo/watch) are `bluer-objects`.
 
@@ -19,4 +19,4 @@ Also home to [blue README](./bluer_objects/README/).
 
 [![pylint](https://github.com/kamangir/bluer-objects/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/bluer-objects/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/bluer-objects/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/bluer-objects.svg)](https://pypi.org/project/bluer-objects/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/bluer-objects)](https://pypistats.org/packages/bluer-objects)
 
-built by 🌀 [`bluer_options-5.24.1`](https://github.com/kamangir/awesome-bash-cli), based on 🌀 [`bluer_objects-6.25.1`](https://github.com/kamangir/bluer-objects).
+built by 🌀 [`bluer_options-5.26.1`](https://github.com/kamangir/awesome-bash-cli), based on 🌀 [`bluer_objects-6.27.1`](https://github.com/kamangir/bluer-objects).

{bluer_objects-6.25.1 → bluer_objects-6.27.1}/bluer_objects/.abcli/alias.sh

@@ -2,7 +2,7 @@
 
 alias @ai=bluer_ai
 
-alias @download=abcli_download
+alias @download=bluer_objects_download
 
 alias @gif=abcli_gif
 
@@ -10,7 +10,7 @@ alias @assets=abcli_assets
 
 alias @cache=abcli_cache
 alias @ref="abcli_cache read"
-alias @tags=abcli_tags
+alias @tags=abcli_mlflow_tags
 
 alias @clone=abcli_clone
 alias @cp=abcli_clone
@@ -33,4 +33,4 @@ alias @publish=bluer_ai_publish
 
 alias @select=bluer_ai_select
 
-alias @upload=abcli_upload
+alias @upload=bluer_objects_upload

{bluer_objects-6.25.1 → bluer_objects-6.27.1}/bluer_objects/.abcli/clone.sh

@@ -22,12 +22,12 @@ function abcli_clone() {
         [[ $? -ne 0 ]] && return 1
 
         [[ "$do_relate" == 1 ]] &&
-            abcli_tags set \
+            abcli_mlflow_tags set \
                 $object_name \
                 cloned.$s3_uri
 
         [[ "$do_upload" == 1 ]] &&
-            abcli_upload - $object_name
+            bluer_objects_upload - $object_name
 
         return 0
     fi
@@ -43,7 +43,7 @@ function abcli_clone() {
     abcli_log "$object_1_name -clone:$transfer_mechanism-> $object_2_name"
 
     [[ "$do_download" == 1 ]] &&
-        abcli_download - $object_1_name
+        bluer_objects_download - $object_1_name
 
     local object_1_path=$ABCLI_OBJECT_ROOT/$object_1_name
     local object_2_path=$ABCLI_OBJECT_ROOT/$object_2_name
@@ -64,12 +64,12 @@ function abcli_clone() {
     fi
 
     [[ "$clone_tags" == 1 ]] &&
-        abcli_tags clone \
+        abcli_mlflow_tags clone \
             $object_1_name \
             $object_2_name
 
     [[ "$do_relate" == 1 ]] &&
-        abcli_tags set \
+        abcli_mlflow_tags set \
             $object_2_name \
             cloned.$object_1_name
 
@@ -88,7 +88,7 @@ function abcli_clone() {
             $object_2_path/metadata-$object_1_name.yaml
 
     [[ "$do_upload" == 1 ]] &&
-        abcli_upload - $object_2_name
+        bluer_objects_upload - $object_2_name
 
     return 0
 }

bluer_objects-6.27.1/bluer_objects/.abcli/download.sh

@@ -0,0 +1,22 @@
+#! /usr/bin/env bash
+
+function bluer_objects_download() {
+    local options=$1
+    local filename=$(abcli_option "$options" filename)
+
+    local object_name=$(abcli_clarify_object $2 .)
+
+    python3 -m bluer_objects.storage \
+        download \
+        --object_name $object_name \
+        --filename "$filename"
+    [[ $? -ne 0 ]] && return 1
+
+    local open_options=$3
+    local do_open=$(abcli_option_int "$open_options" open 0)
+    [[ "$do_open" == 0 ]] &&
+        return 0
+
+    bluer_ai_open filename=$filename,$open_options \
+        $object_name
+}

{bluer_objects-6.25.1 → bluer_objects-6.27.1}/bluer_objects/.abcli/gif.sh

@@ -9,7 +9,7 @@ function abcli_gif() {
     local object_name=$(abcli_clarify_object $2 .)
 
     [[ "$do_download" == 1 ]] &&
-        abcli_download - $object_name
+        bluer_objects_download - $object_name
 
     abcli_log "generating animated gif: $object_name ..."
 
@@ -21,7 +21,7 @@ function abcli_gif() {
     local status="$?"
 
     [[ "$do_upload" == 1 ]] &&
-        abcli_upload - $object_name
+        bluer_objects_upload - $object_name
 
     return $status
 }

{bluer_objects-6.25.1 → bluer_objects-6.27.1}/bluer_objects/.abcli/mlflow.sh

@@ -1,6 +1,6 @@
 #! /usr/bin/env bash
 
-export MLFLOW_TRACKING_URI="databricks"
+export MLFLOW_TRACKING_URI=$HOME/mlruns
 
 function abcli_mlflow() {
     local task=$1

{bluer_objects-6.25.1 → bluer_objects-6.27.1}/bluer_objects/.abcli/publish.sh

@@ -9,16 +9,16 @@ function bluer_ai_publish() {
 
     local object_name=$(abcli_clarify_object $2 .)
     [[ "$do_download" == 1 ]] &&
-        abcli_download - $object_name
+        bluer_objects_download - $object_name
 
-    abcli_tags set $object_name published
+    abcli_mlflow_tags set $object_name published
 
     local public_object_name=$(abcli_option "$options" as $object_name)
 
     if [ "$do_tar" == 1 ]; then
         abcli_log "publishing $object_name -> $public_object_name.tar.gz"
 
-        abcli_upload ~open,solid $object_name
+        bluer_objects_upload ~open,solid $object_name
         aws s3 cp \
             $ABCLI_S3_OBJECT_PREFIX/$object_name.tar.gz \
             s3://$ABCLI_AWS_S3_PUBLIC_BUCKET_NAME/$public_object_name.tar.gz

{bluer_objects-6.25.1 → bluer_objects-6.27.1}/bluer_objects/.abcli/tests/ls.sh

@@ -3,7 +3,7 @@
 function test_bluer_objects_ls() {
     bluer_ai_select
 
-    abcli_upload
+    bluer_objects_upload
 
     bluer_ai_ls cloud
 

bluer_objects-6.27.1/bluer_objects/.abcli/tests/storage.sh

@@ -0,0 +1,60 @@
+#! /usr/bin/env bash
+
+function test_bluer_objects_storage() {
+    local options=$1
+
+    local object_name=test_bluer_objects_storage-$(abcli_string_timestamp_short)
+    local object_path=$ABCLI_OBJECT_ROOT/$object_name
+    mkdir -pv $object_path
+
+    local suffix
+    local extension=yaml
+    for suffix in this.$extension that.$extension subfolder/this.$extension subfolder/that.$extension; do
+        python3 -m bluer_objects.file \
+            create_a_file \
+            --filename $object_path/$suffix
+    done
+
+    # testing upload
+    abcli_hr
+
+    bluer_objects_upload \
+        filename=this.$extension \
+        $object_name
+    [[ $? -ne 0 ]] && return 1
+    abcli_hr
+
+    bluer_objects_upload \
+        filename=subfolder/this.$extension \
+        $object_name
+    [[ $? -ne 0 ]] && return 1
+    abcli_hr
+
+    bluer_objects_upload \
+        - \
+        $object_name
+    [[ $? -ne 0 ]] && return 1
+    abcli_hr
+
+    # clean-up
+    rm -rfv $object_path
+    abcli_hr
+
+    # testing download
+
+    bluer_objects_download \
+        filename=this.$extension \
+        $object_name
+    [[ $? -ne 0 ]] && return 1
+    abcli_hr
+
+    bluer_objects_download \
+        filename=subfolder/this.$extension \
+        $object_name
+    [[ $? -ne 0 ]] && return 1
+    abcli_hr
+
+    bluer_objects_download \
+        - \
+        $object_name
+}

bluer_objects-6.27.1/bluer_objects/.abcli/upload.sh

@@ -0,0 +1,21 @@
+#! /usr/bin/env bash
+
+function bluer_objects_upload() {
+    local options=$1
+    local filename=$(abcli_option "$options" filename)
+
+    local object_name=$(abcli_clarify_object $2 .)
+    local object_path=$ABCLI_OBJECT_ROOT/$object_name
+
+    rm -rf $object_path/auxiliary
+
+    python3 -m bluer_objects.storage \
+        upload \
+        --object_name $object_name \
+        --filename "$filename"
+    [[ $? -ne 0 ]] && return 1
+
+    if [[ -z "$filename" ]]; then
+        abcli_mlflow_log_run $object_name
+    fi
+}

{bluer_objects-6.25.1 → bluer_objects-6.27.1}/bluer_objects/__init__.py

@@ -4,7 +4,7 @@ ICON = "🌀"
 
 DESCRIPTION = f"{ICON} data objects for Bash."
 
-VERSION = "6.25.1"
+VERSION = "6.27.1"
 
 REPO_NAME = "bluer-objects"
 

{bluer_objects-6.25.1 → bluer_objects-6.27.1}/bluer_objects/config.env

@@ -4,4 +4,6 @@ DATABRICKS_WORKSPACE=abcli-v4
 
 ABCLI_MLFLOW_URL_SUBDOMAIN=ml
 
-ABCLI_MLFLOW_EXPERIMENT_PREFIX=/Users/kamangirblog@gmail.com/
+ABCLI_MLFLOW_EXPERIMENT_PREFIX=experiment/
+
+BLUER_OBJECTS_STORAGE_INTERFACE=webdavzip

{bluer_objects-6.25.1 → bluer_objects-6.27.1}/bluer_objects/env.py

@@ -42,5 +42,10 @@ DATABRICKS_TOKEN = get_env("DATABRICKS_TOKEN")
 
 ABCLI_MLFLOW_EXPERIMENT_PREFIX = get_env("ABCLI_MLFLOW_EXPERIMENT_PREFIX")
 
-
 ABCLI_S3_OBJECT_PREFIX = "to-be-removed"
+
+BLUER_OBJECTS_STORAGE_INTERFACE = get_env("BLUER_OBJECTS_STORAGE_INTERFACE")
+
+WEBDAV_HOSTNAME = get_env("WEBDAV_HOSTNAME")
+WEBDAV_LOGIN = get_env("WEBDAV_LOGIN")
+WEBDAV_PASSWORD = get_env("WEBDAV_PASSWORD")

{bluer_objects-6.25.1 → bluer_objects-6.27.1}/bluer_objects/file/__main__.py

@@ -14,7 +14,7 @@ parser = argparse.ArgumentParser(NAME)
 parser.add_argument(
     "task",
     type=str,
-    help="replace|size",
+    help="create_a_file | replace | size",
 )
 parser.add_argument(
     "--filename",
@@ -30,10 +30,23 @@ parser.add_argument(
     type=str,
     help="<that-1+that-2+that-3>",
 )
+parser.add_argument(
+    "--size",
+    type=int,
+    default=16,
+)
 args = parser.parse_args()
 
 success = False
-if args.task == "replace":
+if args.task == "create_a_file":
+    success = file.save_yaml(
+        args.filename,
+        {
+            string.random(length=args.size): string.random(length=args.size)
+            for _ in range(args.size)
+        },
+    )
+elif args.task == "replace":
     logger.info(f"{NAME}.{args.task}: {args.this} -> {args.that} in {args.filename}")
 
     success, content = file.load_text(args.filename)

bluer_objects-6.27.1/bluer_objects/help/download.py

@@ -0,0 +1,23 @@
+from typing import List
+
+from bluer_options.terminal import show_usage, xtra
+
+
+def help_download(
+    tokens: List[str],
+    mono: bool,
+) -> str:
+    options = "filename=<filename>"
+
+    open_options = "open,QGIS"
+
+    return show_usage(
+        [
+            "@download",
+            f"[{options}]",
+            "[.|<object-name>]",
+            f"[{open_options}]",
+        ],
+        "download <object-name>.",
+        mono=mono,
+    )

bluer_objects-6.27.1/bluer_objects/help/functions.py

@@ -0,0 +1,14 @@
+from bluer_ai.help.generic import help_functions as generic_help_functions
+
+from bluer_objects import ALIAS
+from bluer_objects.help.download import help_download
+from bluer_objects.help.upload import help_upload
+
+help_functions = generic_help_functions(plugin_name=ALIAS)
+
+help_functions.update(
+    {
+        "download": help_download,
+        "upload": help_upload,
+    }
+)

bluer_objects-6.27.1/bluer_objects/help/upload.py

@@ -0,0 +1,20 @@
+from typing import List
+
+from bluer_options.terminal import show_usage, xtra
+
+
+def help_upload(
+    tokens: List[str],
+    mono: bool,
+) -> str:
+    options = "filename=<filename>"
+
+    return show_usage(
+        [
+            "@upload",
+            f"[{options}]",
+            "[.|<object-name>]",
+        ],
+        "upload <object-name>.",
+        mono=mono,
+    )

bluer_objects-6.27.1/bluer_objects/host/__init__.py

@@ -0,0 +1,12 @@
+from typing import List
+
+from bluer_ai.host import signature as bluer_ai_signature
+
+from bluer_objects import fullname
+from bluer_objects.host.functions import shell, unzip, zip
+
+
+def signature() -> List[str]:
+    return [
+        fullname(),
+    ] + bluer_ai_signature()

{bluer_objects-6.25.1 → bluer_objects-6.27.1}/bluer_objects/host/__main__.py

@@ -15,7 +15,6 @@ parser = argparse.ArgumentParser(NAME)
 parser.add_argument(
     "task",
     type=str,
-    default="get",
     help="add_signature",
 )
 parser.add_argument(

{bluer_objects-6.25.1 → bluer_objects-6.27.1}/bluer_objects/host/functions.py

@@ -61,6 +61,19 @@ def unzip(
         output_folder = file.path(zip_filename)
 
     return shell(
-        command=f'unzip -q -o "{zip_filename}" -d "{output_folder}"',
+        command=f'unzip -o "{zip_filename}" -d "{output_folder}"',
+        log=log,
+    )
+
+
+def zip(
+    zip_filename: str,
+    input_folder: str = "",
+    work_dir: str = ".",
+    log: bool = False,
+) -> bool:
+    return shell(
+        command=f'zip -r "{zip_filename}" "{input_folder}"',
+        work_dir=work_dir,
         log=log,
     )

{bluer_objects-6.25.1 → bluer_objects-6.27.1}/bluer_objects/metadata/get.py

@@ -2,7 +2,7 @@ from typing import Any
 
 
 from bluer_objects import file
-from bluer_objects import objects
+from bluer_objects import storage
 from bluer_objects.metadata.enums import MetadataSourceType
 
 
@@ -59,7 +59,7 @@ def get_from_object(
     download: bool = False,
     **kwargs,
 ) -> Any:
-    if download and not objects.download(
+    if download and not storage.download(
         object_name=object_name,
         filename="metadata.yaml",
     ):

{bluer_objects-6.25.1 → bluer_objects-6.27.1}/bluer_objects/metadata/post.py

@@ -7,6 +7,7 @@ from blueness import module
 from bluer_objects.metadata.enums import MetadataSourceType
 from bluer_objects import NAME, file
 from bluer_objects import objects
+from bluer_objects import storage
 from bluer_objects.logger import logger
 
 NAME = module.name(__file__, NAME)
@@ -65,7 +66,7 @@ def post_to_object(
     upload: bool = False,
     **kwargs,
 ) -> bool:
-    if download and not objects.download(
+    if download and not storage.download(
         object_name=object_name,
         filename="metadata.yaml",
     ):

bluer_objects-6.27.1/bluer_objects/sample.env

@@ -0,0 +1,3 @@
+WEBDAV_HOSTNAME=
+WEBDAV_LOGIN=
+WEBDAV_PASSWORD=

bluer_objects-6.27.1/bluer_objects/storage/WebDAV.py

@@ -0,0 +1,114 @@
+from webdav3.client import Client
+
+from bluer_objects.storage.base import StorageInterface
+from bluer_objects import env, file, path
+from bluer_objects import objects
+from bluer_objects.logger import logger
+
+
+# https://chatgpt.com/c/67e06812-4af0-8005-b2ab-5f9a1eabbbe3
+# https://pypi.org/project/webdavclient3/
+
+
+# ❗️ fails on .txt, .json, .yaml, ... other files with 'content-length',
+# see WebDAV.ipynb for details (set BLUER_OBJECTS_STORAGE_INTERFACE=webdav).
+class WebDAVInterface(StorageInterface):
+    name = "webdav"
+
+    def __init__(self):
+        super().__init__()
+
+        config = {
+            "webdav_hostname": env.WEBDAV_HOSTNAME,
+            "webdav_login": env.WEBDAV_LOGIN,
+            "webdav_password": env.WEBDAV_PASSWORD,
+        }
+
+        self.client = Client(config)
+
+    def mkdir(
+        self,
+        path: str,
+        log: bool = True,
+    ) -> bool:
+        try:
+            if self.client.check(path):
+                return True
+
+            self.client.mkdir(path)
+        except Exception as e:
+            logger.error(e)
+            return False
+
+        if log:
+            logger.info(
+                "{}.mkdir: {}".format(
+                    self.__class__.__name__,
+                    path,
+                )
+            )
+
+        return True
+
+    def download(
+        self,
+        object_name: str,
+        filename: str = "",
+        log: bool = True,
+    ) -> bool:
+        local_path = objects.path_of(
+            object_name=object_name,
+            filename=filename,
+            create=True,
+        )
+
+        if filename:
+            if not path.create(file.path(local_path)):
+                return False
+
+        try:
+            self.client.download_sync(
+                remote_path=f"{object_name}/{filename}",
+                local_path=local_path,
+            )
+        except Exception as e:
+            logger.error(e)
+            return False
+
+        return super().download(
+            object_name=object_name,
+            filename=filename,
+            log=log,
+        )
+
+    def upload(
+        self,
+        object_name: str,
+        filename: str = "",
+        log: bool = True,
+    ) -> bool:
+        if filename:
+            remote_dir = "/".join([object_name] + filename.split("/")[:-1])
+            if not self.mkdir(
+                path=remote_dir,
+                log=log,
+            ):
+                return False
+
+        try:
+            self.client.upload_sync(
+                remote_path=f"{object_name}/{filename}",
+                local_path=objects.path_of(
+                    object_name=object_name,
+                    filename=filename,
+                ),
+            )
+        except Exception as e:
+            logger.error(e)
+            return False
+
+        return super().upload(
+            object_name=object_name,
+            filename=filename,
+            log=log,
+        )