bluer-objects 6.24.1__py3-none-any.whl → 6.26.1__py3-none-any.whl

bluer_objects/.abcli/alias.sh

@@ -2,7 +2,7 @@
 
 alias @ai=bluer_ai
 
-alias @download=abcli_download
+alias @download=bluer_objects_download
 
 alias @gif=abcli_gif
 
@@ -10,7 +10,7 @@ alias @assets=abcli_assets
 
 alias @cache=abcli_cache
 alias @ref="abcli_cache read"
-alias @tags=abcli_tags
+alias @tags=abcli_mlflow_tags
 
 alias @clone=abcli_clone
 alias @cp=abcli_clone
@@ -33,4 +33,4 @@ alias @publish=bluer_ai_publish
 
 alias @select=bluer_ai_select
 
-alias @upload=abcli_upload
+alias @upload=bluer_objects_upload

bluer_objects/.abcli/clone.sh

@@ -22,12 +22,12 @@ function abcli_clone() {
         [[ $? -ne 0 ]] && return 1
 
         [[ "$do_relate" == 1 ]] &&
-            abcli_tags set \
+            abcli_mlflow_tags set \
                 $object_name \
                 cloned.$s3_uri
 
         [[ "$do_upload" == 1 ]] &&
-            abcli_upload - $object_name
+            bluer_objects_upload - $object_name
 
         return 0
     fi
@@ -43,7 +43,7 @@ function abcli_clone() {
     abcli_log "$object_1_name -clone:$transfer_mechanism-> $object_2_name"
 
     [[ "$do_download" == 1 ]] &&
-        abcli_download - $object_1_name
+        bluer_objects_download - $object_1_name
 
     local object_1_path=$ABCLI_OBJECT_ROOT/$object_1_name
     local object_2_path=$ABCLI_OBJECT_ROOT/$object_2_name
@@ -64,12 +64,12 @@ function abcli_clone() {
     fi
 
     [[ "$clone_tags" == 1 ]] &&
-        abcli_tags clone \
+        abcli_mlflow_tags clone \
             $object_1_name \
             $object_2_name
 
     [[ "$do_relate" == 1 ]] &&
-        abcli_tags set \
+        abcli_mlflow_tags set \
             $object_2_name \
             cloned.$object_1_name
 
@@ -88,7 +88,7 @@ function abcli_clone() {
             $object_2_path/metadata-$object_1_name.yaml
 
     [[ "$do_upload" == 1 ]] &&
-        abcli_upload - $object_2_name
+        bluer_objects_upload - $object_2_name
 
     return 0
 }

bluer_objects/.abcli/download.sh

@@ -1,53 +1,22 @@
 #! /usr/bin/env bash
 
-function abcli_download() {
+function bluer_objects_download() {
     local options=$1
     local filename=$(abcli_option "$options" filename)
 
     local object_name=$(abcli_clarify_object $2 .)
-    local object_path=$ABCLI_OBJECT_ROOT/$object_name
 
-    local open_options=$3
-
-    if [ -f "../$object_name.tar.gz" ]; then
-        abcli_log "✅ $object_name.tar.gz already exists - skipping download."
-        return 1
-    fi
-
-    if [ ! -z "$filename" ]; then
-        abcli_log "downloading $object_name/$filename ..."
-        aws s3 cp "$ABCLI_S3_OBJECT_PREFIX/$object_name/$filename" "$object_path/$filename"
-    else
-        local exists=$(aws s3 ls $ABCLI_AWS_S3_BUCKET_NAME/$ABCLI_AWS_S3_PREFIX/$object_name.tar.gz)
-        if [ -z "$exists" ]; then
-            abcli_log "downloading $object_name ..."
-
-            aws s3 sync \
-                "$ABCLI_S3_OBJECT_PREFIX/$object_name" \
-                "$object_path" \
-                --exact-timestamps
-        else
-            abcli_log "downloading $object_name.tar.gz ..."
-
-            pushd $ABCLI_OBJECT_ROOT >/dev/null
-
-            aws s3 cp \
-                "$ABCLI_S3_OBJECT_PREFIX/$object_name.tar.gz" .
-
-            local object_size=$(bluer_objects_file size $object_name.tar.gz)
-            abcli_log "$object_name.tar.gz ($object_size) downloaded."
-
-            tar -xvf "$object_name.tar.gz"
-
-            popd >/dev/null
-        fi
-
-    fi
+    python3 -m bluer_objects.storage \
+        download \
+        --object_name $object_name \
+        --filename "$filename"
+    [[ $? -ne 0 ]] && return 1
 
+    local open_options=$3
     local do_open=$(abcli_option_int "$open_options" open 0)
-    [[ "$do_open" == 1 ]] &&
-        abcli_open filename=$filename,$open_options \
-            $object_name
+    [[ "$do_open" == 0 ]] &&
+        return 0
 
-    return 0
+    bluer_ai_open filename=$filename,$open_options \
+        $object_name
 }

bluer_objects/.abcli/gif.sh

@@ -9,7 +9,7 @@ function abcli_gif() {
     local object_name=$(abcli_clarify_object $2 .)
 
     [[ "$do_download" == 1 ]] &&
-        abcli_download - $object_name
+        bluer_objects_download - $object_name
 
     abcli_log "generating animated gif: $object_name ..."
 
@@ -21,7 +21,7 @@ function abcli_gif() {
     local status="$?"
 
     [[ "$do_upload" == 1 ]] &&
-        abcli_upload - $object_name
+        bluer_objects_upload - $object_name
 
     return $status
 }

bluer_objects/.abcli/mlflow.sh

@@ -1,6 +1,6 @@
 #! /usr/bin/env bash
 
-export MLFLOW_TRACKING_URI="databricks"
+export MLFLOW_TRACKING_URI=$abcli_path_git/mlruns
 
 function abcli_mlflow() {
     local task=$1

bluer_objects/.abcli/publish.sh

@@ -9,16 +9,16 @@ function bluer_ai_publish() {
 
     local object_name=$(abcli_clarify_object $2 .)
     [[ "$do_download" == 1 ]] &&
-        abcli_download - $object_name
+        bluer_objects_download - $object_name
 
-    abcli_tags set $object_name published
+    abcli_mlflow_tags set $object_name published
 
     local public_object_name=$(abcli_option "$options" as $object_name)
 
     if [ "$do_tar" == 1 ]; then
         abcli_log "publishing $object_name -> $public_object_name.tar.gz"
 
-        abcli_upload ~open,solid $object_name
+        bluer_objects_upload ~open,solid $object_name
         aws s3 cp \
             $ABCLI_S3_OBJECT_PREFIX/$object_name.tar.gz \
             s3://$ABCLI_AWS_S3_PUBLIC_BUCKET_NAME/$public_object_name.tar.gz

bluer_objects/.abcli/tests/ls.sh

@@ -3,7 +3,7 @@
 function test_bluer_objects_ls() {
     bluer_ai_select
 
-    abcli_upload
+    bluer_objects_upload
 
     bluer_ai_ls cloud
 

bluer_objects/.abcli/tests/storage.sh

@@ -0,0 +1,60 @@
+#! /usr/bin/env bash
+
+function test_bluer_objects_storage() {
+    local options=$1
+
+    local object_name=test_bluer_objects_storage-$(abcli_string_timestamp_short)
+    local object_path=$ABCLI_OBJECT_ROOT/$object_name
+    mkdir -pv $object_path
+
+    local suffix
+    local extension=yaml
+    for suffix in this.$extension that.$extension subfolder/this.$extension subfolder/that.$extension; do
+        python3 -m bluer_objects.file \
+            create_a_file \
+            --filename $object_path/$suffix
+    done
+
+    # testing upload
+    abcli_hr
+
+    bluer_objects_upload \
+        filename=this.$extension \
+        $object_name
+    [[ $? -ne 0 ]] && return 1
+    abcli_hr
+
+    bluer_objects_upload \
+        filename=subfolder/this.$extension \
+        $object_name
+    [[ $? -ne 0 ]] && return 1
+    abcli_hr
+
+    bluer_objects_upload \
+        - \
+        $object_name
+    [[ $? -ne 0 ]] && return 1
+    abcli_hr
+
+    # clean-up
+    rm -rfv $object_path
+    abcli_hr
+
+    # testing download
+
+    bluer_objects_download \
+        filename=this.$extension \
+        $object_name
+    [[ $? -ne 0 ]] && return 1
+    abcli_hr
+
+    bluer_objects_download \
+        filename=subfolder/this.$extension \
+        $object_name
+    [[ $? -ne 0 ]] && return 1
+    abcli_hr
+
+    bluer_objects_download \
+        - \
+        $object_name
+}

bluer_objects/.abcli/upload.sh

@@ -1,73 +1,21 @@
 #! /usr/bin/env bash
 
-function abcli_upload() {
+function bluer_objects_upload() {
     local options=$1
     local filename=$(abcli_option "$options" filename)
-    local do_open=$(abcli_option_int "$options" open 1)
-    local do_solid=$(abcli_option_int "$options" solid 0)
-    local warn_if_exists=$(abcli_option_int "$options" warn_if_exists 1)
-    local log_to_mlflow=$(abcli_option_int "$options" mlflow 0)
-    local no_mlflow=$(abcli_option_int "$options" no_mlflow 0)
 
     local object_name=$(abcli_clarify_object $2 .)
     local object_path=$ABCLI_OBJECT_ROOT/$object_name
 
-    # https://stackoverflow.com/a/45200066
-    local exists=$(aws s3 ls $ABCLI_AWS_S3_BUCKET_NAME/$ABCLI_AWS_S3_PREFIX/$object_name.tar.gz)
-    if [ ! -z "$exists" ]; then
-        if [[ "$warn_if_exists" == 1 ]]; then
-            abcli_log_warning "@abcli: upload: $object_name.tar.gz already exists on the cloud, use \"abcli object open\" to open the object."
-        else
-            abcli_log "✅ ☁️  $object_name.tar.gz."
-        fi
-        return
-    fi
-
     rm -rf $object_path/auxiliary
 
-    if [ ! -z "$filename" ]; then
-        local file_size=$(bluer_objects_file size $filename)
-        abcli_log "uploading $object_name/$filename ($file_size) ..."
-
-        aws s3 cp \
-            $object_path/$filename \
-            $ABCLI_S3_OBJECT_PREFIX/$object_name/
-
-        return
-    fi
-
-    if [ "$do_open" == 1 ]; then
-        abcli_log "uploading $object_name ..."
-
-        aws s3 sync \
-            $object_path/ \
-            $ABCLI_S3_OBJECT_PREFIX/$object_name/
-
-        abcli_tags set $object_name open
-    fi
-
-    if [ "$do_solid" == 1 ]; then
-        pushd $ABCLI_OBJECT_ROOT >/dev/null
-
-        tar -czvf \
-            $object_name.tar.gz \
-            ./$object_name
-
-        local object_size=$(bluer_objects_file size $object_path.tar.gz)
-        abcli_log "uploading $object_name.tar.gz ($object_size) ..."
-
-        aws s3 cp \
-            $object_name.tar.gz \
-            $ABCLI_S3_OBJECT_PREFIX/
-
-        abcli_tags set $object_name solid
-
-        popd >/dev/null
-    fi
+    python3 -m bluer_objects.storage \
+        upload \
+        --object_name $object_name \
+        --filename "$filename"
+    [[ $? -ne 0 ]] && return 1
 
-    if [[ "$log_to_mlflow" == 1 ]]; then
-        abcli_mlflow log_artifacts $object_name
-    elif [[ "$no_mlflow" == 0 ]]; then
-        abcli_mlflow log_run $object_name
+    if [[ -z "$filename" ]]; then
+        abcli_mlflow_log_run $object_name
     fi
 }

bluer_objects/__init__.py

@@ -4,7 +4,7 @@ ICON = "🌀"
 
 DESCRIPTION = f"{ICON} data objects for Bash."
 
-VERSION = "6.24.1"
+VERSION = "6.26.1"
 
 REPO_NAME = "bluer-objects"
 

bluer_objects/config.env

@@ -4,4 +4,6 @@ DATABRICKS_WORKSPACE=abcli-v4
 
 ABCLI_MLFLOW_URL_SUBDOMAIN=ml
 
-ABCLI_MLFLOW_EXPERIMENT_PREFIX=/Users/kamangirblog@gmail.com/
+ABCLI_MLFLOW_EXPERIMENT_PREFIX=experiment/
+
+BLUER_OBJECTS_STORAGE_INTERFACE=webdavzip

bluer_objects/env.py

@@ -42,5 +42,10 @@ DATABRICKS_TOKEN = get_env("DATABRICKS_TOKEN")
 
 ABCLI_MLFLOW_EXPERIMENT_PREFIX = get_env("ABCLI_MLFLOW_EXPERIMENT_PREFIX")
 
-
 ABCLI_S3_OBJECT_PREFIX = "to-be-removed"
+
+BLUER_OBJECTS_STORAGE_INTERFACE = get_env("BLUER_OBJECTS_STORAGE_INTERFACE")
+
+WEBDAV_HOSTNAME = get_env("WEBDAV_HOSTNAME")
+WEBDAV_LOGIN = get_env("WEBDAV_LOGIN")
+WEBDAV_PASSWORD = get_env("WEBDAV_PASSWORD")

bluer_objects/file/__main__.py

@@ -14,7 +14,7 @@ parser = argparse.ArgumentParser(NAME)
 parser.add_argument(
     "task",
     type=str,
-    help="replace|size",
+    help="create_a_file | replace | size",
 )
 parser.add_argument(
     "--filename",
@@ -30,10 +30,23 @@ parser.add_argument(
     type=str,
     help="<that-1+that-2+that-3>",
 )
+parser.add_argument(
+    "--size",
+    type=int,
+    default=16,
+)
 args = parser.parse_args()
 
 success = False
-if args.task == "replace":
+if args.task == "create_a_file":
+    success = file.save_yaml(
+        args.filename,
+        {
+            string.random(length=args.size): string.random(length=args.size)
+            for _ in range(args.size)
+        },
+    )
+elif args.task == "replace":
     logger.info(f"{NAME}.{args.task}: {args.this} -> {args.that} in {args.filename}")
 
     success, content = file.load_text(args.filename)

bluer_objects/help/download.py

@@ -0,0 +1,23 @@
+from typing import List
+
+from bluer_options.terminal import show_usage, xtra
+
+
+def help_download(
+    tokens: List[str],
+    mono: bool,
+) -> str:
+    options = "filename=<filename>"
+
+    open_options = "open,QGIS"
+
+    return show_usage(
+        [
+            "@download",
+            f"[{options}]",
+            "[.|<object-name>]",
+            f"[{open_options}]",
+        ],
+        "download <object-name>.",
+        mono=mono,
+    )

bluer_objects/help/functions.py

@@ -1,5 +1,14 @@
-from abcli.help.generic import help_functions as generic_help_functions
+from bluer_ai.help.generic import help_functions as generic_help_functions
 
 from bluer_objects import ALIAS
+from bluer_objects.help.download import help_download
+from bluer_objects.help.upload import help_upload
 
 help_functions = generic_help_functions(plugin_name=ALIAS)
+
+help_functions.update(
+    {
+        "download": help_download,
+        "upload": help_upload,
+    }
+)

bluer_objects/help/upload.py

@@ -0,0 +1,20 @@
+from typing import List
+
+from bluer_options.terminal import show_usage, xtra
+
+
+def help_upload(
+    tokens: List[str],
+    mono: bool,
+) -> str:
+    options = "filename=<filename>"
+
+    return show_usage(
+        [
+            "@upload",
+            f"[{options}]",
+            "[.|<object-name>]",
+        ],
+        "upload <object-name>.",
+        mono=mono,
+    )

bluer_objects/host/__init__.py

@@ -1 +1,12 @@
-from bluer_objects.host.functions import shell, unzip
+from typing import List
+
+from bluer_ai.host import signature as bluer_ai_signature
+
+from bluer_objects import fullname
+from bluer_objects.host.functions import shell, unzip, zip
+
+
+def signature() -> List[str]:
+    return [
+        fullname(),
+    ] + bluer_ai_signature()

bluer_objects/host/__main__.py

@@ -15,7 +15,6 @@ parser = argparse.ArgumentParser(NAME)
 parser.add_argument(
     "task",
     type=str,
-    default="get",
     help="add_signature",
 )
 parser.add_argument(

bluer_objects/host/functions.py

@@ -61,6 +61,19 @@ def unzip(
         output_folder = file.path(zip_filename)
 
     return shell(
-        command=f'unzip -q -o "{zip_filename}" -d "{output_folder}"',
+        command=f'unzip -o "{zip_filename}" -d "{output_folder}"',
+        log=log,
+    )
+
+
+def zip(
+    zip_filename: str,
+    input_folder: str = "",
+    work_dir: str = ".",
+    log: bool = False,
+) -> bool:
+    return shell(
+        command=f'zip -r "{zip_filename}" "{input_folder}"',
+        work_dir=work_dir,
         log=log,
     )

bluer_objects/metadata/get.py

@@ -2,7 +2,7 @@ from typing import Any
 
 
 from bluer_objects import file
-from bluer_objects import objects
+from bluer_objects import storage
 from bluer_objects.metadata.enums import MetadataSourceType
 
 
@@ -59,7 +59,7 @@ def get_from_object(
     download: bool = False,
     **kwargs,
 ) -> Any:
-    if download and not objects.download(
+    if download and not storage.download(
         object_name=object_name,
         filename="metadata.yaml",
     ):

bluer_objects/metadata/post.py

@@ -7,6 +7,7 @@ from blueness import module
 from bluer_objects.metadata.enums import MetadataSourceType
 from bluer_objects import NAME, file
 from bluer_objects import objects
+from bluer_objects import storage
 from bluer_objects.logger import logger
 
 NAME = module.name(__file__, NAME)
@@ -65,7 +66,7 @@ def post_to_object(
     upload: bool = False,
     **kwargs,
 ) -> bool:
-    if download and not objects.download(
+    if download and not storage.download(
         object_name=object_name,
         filename="metadata.yaml",
     ):

bluer_objects/sample.env

@@ -1,16 +1,3 @@
-# to complete the `rds` section create a database as follows:
-# go to `AWS Management Console` -> `RDS` -> `Create database`.
-# use `Standard Create` -> `MySQL` -> `MySQL 8.0.20` -> `Dev/Test`.
-# set `DB instance identifier` to `abcli`.
-# set `Credentials Settings` -> `Master username` and `Master Password` and update `host`, `username`, and `password` here.
-# set `DB instance size` to `Burstable Classes` and select `db.t3.small`.
-# set `Storage` to `20` and disable `Storage autoscaling`.
-# in `Connectivity` set `Public Access` to `Yes`.
-# in `Additional Configuration` set `initial database name` to `abcli`.
-ABCLI_AWS_RDS_HOST=
-ABCLI_AWS_RDS_PASSWORD=
-
-# https://docs.databricks.com/en/mlflow/tracking.html
-# https://docs.databricks.com/en/dev-tools/auth/pat.html#databricks-personal-access-tokens-for-workspace-users
-DATABRICKS_HOST=
-DATABRICKS_TOKEN=
+WEBDAV_HOSTNAME=
+WEBDAV_LOGIN=
+WEBDAV_PASSWORD=

bluer_objects/storage/WebDAV.py

@@ -0,0 +1,114 @@
+from webdav3.client import Client
+
+from bluer_objects.storage.base import StorageInterface
+from bluer_objects import env, file, path
+from bluer_objects import objects
+from bluer_objects.logger import logger
+
+
+# https://chatgpt.com/c/67e06812-4af0-8005-b2ab-5f9a1eabbbe3
+# https://pypi.org/project/webdavclient3/
+
+
+# ❗️ fails on .txt, .json, .yaml, ... other files with 'content-length',
+# see WebDAV.ipynb for details (set BLUER_OBJECTS_STORAGE_INTERFACE=webdav).
+class WebDAVInterface(StorageInterface):
+    name = "webdav"
+
+    def __init__(self):
+        super().__init__()
+
+        config = {
+            "webdav_hostname": env.WEBDAV_HOSTNAME,
+            "webdav_login": env.WEBDAV_LOGIN,
+            "webdav_password": env.WEBDAV_PASSWORD,
+        }
+
+        self.client = Client(config)
+
+    def mkdir(
+        self,
+        path: str,
+        log: bool = True,
+    ) -> bool:
+        try:
+            if self.client.check(path):
+                return True
+
+            self.client.mkdir(path)
+        except Exception as e:
+            logger.error(e)
+            return False
+
+        if log:
+            logger.info(
+                "{}.mkdir: {}".format(
+                    self.__class__.__name__,
+                    path,
+                )
+            )
+
+        return True
+
+    def download(
+        self,
+        object_name: str,
+        filename: str = "",
+        log: bool = True,
+    ) -> bool:
+        local_path = objects.path_of(
+            object_name=object_name,
+            filename=filename,
+            create=True,
+        )
+
+        if filename:
+            if not path.create(file.path(local_path)):
+                return False
+
+        try:
+            self.client.download_sync(
+                remote_path=f"{object_name}/{filename}",
+                local_path=local_path,
+            )
+        except Exception as e:
+            logger.error(e)
+            return False
+
+        return super().download(
+            object_name=object_name,
+            filename=filename,
+            log=log,
+        )
+
+    def upload(
+        self,
+        object_name: str,
+        filename: str = "",
+        log: bool = True,
+    ) -> bool:
+        if filename:
+            remote_dir = "/".join([object_name] + filename.split("/")[:-1])
+            if not self.mkdir(
+                path=remote_dir,
+                log=log,
+            ):
+                return False
+
+        try:
+            self.client.upload_sync(
+                remote_path=f"{object_name}/{filename}",
+                local_path=objects.path_of(
+                    object_name=object_name,
+                    filename=filename,
+                ),
+            )
+        except Exception as e:
+            logger.error(e)
+            return False
+
+        return super().upload(
+            object_name=object_name,
+            filename=filename,
+            log=log,
+        )