bluer-objects 6.3.1__py3-none-any.whl

bluer_objects/.abcli/abcli.sh

@@ -0,0 +1,9 @@
+#! /usr/bin/env bash
+
+abcli_source_caller_suffix_path /tests
+
+abcli_env_dot_load \
+    caller,ssm,plugin=bluer_objects,suffix=/../..
+
+abcli_env_dot_load \
+    caller,filename=config.env,suffix=/..

bluer_objects/.abcli/actions.sh

@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+function bluer_objects_action_git_before_push() {
+    bluer_objects build_README
+    [[ $? -ne 0 ]] && return 1
+
+    [[ "$(abcli_git get_branch)" != "main" ]] &&
+        return 0
+
+    bluer_objects pypi build
+}

bluer_objects/.abcli/aka.sh

@@ -0,0 +1,3 @@
+#! /usr/bin/env bash
+
+

bluer_objects/.abcli/alias.sh

@@ -0,0 +1,36 @@
+#! /usr/bin/env bash
+
+alias @download=abcli_download
+
+alias @gif=abcli_gif
+
+alias @assets=abcli_assets
+
+alias @cache=abcli_cache
+alias @ref="abcli_cache read"
+alias @tags=abcli_tags
+
+alias @clone=abcli_clone
+alias @cp=abcli_clone
+
+alias @file=bluer_objects_file
+
+alias @host=abcli_host
+
+alias @ls=abcli_ls
+
+alias @metadata=abcli_metadata
+
+alias @mlflow=abcli_mlflow
+
+alias @mysql=abcli_mysql
+
+alias @object=abcli_object
+
+alias @objects=bluer_objects
+
+alias @publish=abcli_publish
+
+alias @select=abcli_select
+
+alias @upload=abcli_upload

bluer_objects/.abcli/blue_objects.sh

@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+function bluer_objects() {
+    local task=$(abcli_unpack_keyword $1 help)
+
+    abcli_generic_task \
+        plugin=bluer_objects,task=$task \
+        "${@:2}"
+}
+
+abcli_log $(bluer_objects version --show_icon 1)

bluer_objects/.abcli/cache.sh

@@ -0,0 +1,5 @@
+#! /usr/bin/env bash
+
+function abcli_cache() {
+    abcli_mlflow_cache "$@"
+}

bluer_objects/.abcli/clone.sh

@@ -0,0 +1,94 @@
+#! /usr/bin/env bash
+
+function abcli_clone() {
+    local options=$1
+    local do_relate=$(abcli_option_int "$options" relate 1)
+    local do_upload=$(abcli_option_int "$options" upload 0)
+
+    local from_s3=$(abcli_option_int "$options" s3 0)
+    if [[ "$from_s3" == 1 ]]; then
+        local s3_uri=${2:-void}
+
+        local object_name=$(abcli_clarify_object $3 .)
+        local object_path=$ABCLI_OBJECT_ROOT/$object_name
+        mkdir -pv $object_path
+
+        abcli_log "$s3_uri -clone-> $object_name"
+
+        aws s3 sync \
+            "$s3_uri" \
+            "$object_path" \
+            --exact-timestamps
+        [[ $? -ne 0 ]] && return 1
+
+        [[ "$do_relate" == 1 ]] &&
+            abcli_tags set \
+                $object_name \
+                cloned.$s3_uri
+
+        [[ "$do_upload" == 1 ]] &&
+            abcli_upload - $object_name
+
+        return 0
+    fi
+
+    local clone_tags=$(abcli_option_int "$options" tags 1)
+    local copy_content=$(abcli_option_int "$options" content 1)
+    local do_download=$(abcli_option_int "$options" download 1)
+    local transfer_mechanism=$(abcli_option_choice "$options" cp,mv mv)
+
+    local object_1_name=$(abcli_clarify_object $2 ..)
+    local object_2_name=$(abcli_clarify_object $3 .)
+
+    abcli_log "$object_1_name -clone:$transfer_mechanism-> $object_2_name"
+
+    [[ "$do_download" == 1 ]] &&
+        abcli_download - $object_1_name
+
+    local object_1_path=$ABCLI_OBJECT_ROOT/$object_1_name
+    local object_2_path=$ABCLI_OBJECT_ROOT/$object_2_name
+
+    if [[ "$copy_content" == 1 ]]; then
+        abcli_eval - \
+            rsync \
+            -avv \
+            $object_1_path/ \
+            $object_2_path
+    else
+        local extension
+        for extension in qgz; do
+            cp -v \
+                $object_1_path/*.$extension \
+                $object_2_path
+        done
+    fi
+
+    [[ "$clone_tags" == 1 ]] &&
+        abcli_tags clone \
+            $object_1_name \
+            $object_2_name
+
+    [[ "$do_relate" == 1 ]] &&
+        abcli_tags set \
+            $object_2_name \
+            cloned.$object_1_name
+
+    pushd $object_2_path >/dev/null
+    local filename
+    for filename in $object_1_name.*; do
+        $transfer_mechanism -v \
+            $filename \
+            $object_2_path/$object_2_name.${filename##*.}
+    done
+    popd >/dev/null
+
+    [[ -f "$object_1_path/metadata.yaml" ]] &&
+        cp -v \
+            $object_1_path/metadata.yaml \
+            $object_2_path/metadata-$object_1_name.yaml
+
+    [[ "$do_upload" == 1 ]] &&
+        abcli_upload - $object_2_name
+
+    return 0
+}

bluer_objects/.abcli/download.sh

@@ -0,0 +1,53 @@
+#! /usr/bin/env bash
+
+function abcli_download() {
+    local options=$1
+    local filename=$(abcli_option "$options" filename)
+
+    local object_name=$(abcli_clarify_object $2 .)
+    local object_path=$ABCLI_OBJECT_ROOT/$object_name
+
+    local open_options=$3
+
+    if [ -f "../$object_name.tar.gz" ]; then
+        abcli_log "✅ $object_name.tar.gz already exists - skipping download."
+        return 1
+    fi
+
+    if [ ! -z "$filename" ]; then
+        abcli_log "downloading $object_name/$filename ..."
+        aws s3 cp "$ABCLI_S3_OBJECT_PREFIX/$object_name/$filename" "$object_path/$filename"
+    else
+        local exists=$(aws s3 ls $ABCLI_AWS_S3_BUCKET_NAME/$ABCLI_AWS_S3_PREFIX/$object_name.tar.gz)
+        if [ -z "$exists" ]; then
+            abcli_log "downloading $object_name ..."
+
+            aws s3 sync \
+                "$ABCLI_S3_OBJECT_PREFIX/$object_name" \
+                "$object_path" \
+                --exact-timestamps
+        else
+            abcli_log "downloading $object_name.tar.gz ..."
+
+            pushd $ABCLI_OBJECT_ROOT >/dev/null
+
+            aws s3 cp \
+                "$ABCLI_S3_OBJECT_PREFIX/$object_name.tar.gz" .
+
+            local object_size=$(bluer_objects_file size $object_name.tar.gz)
+            abcli_log "$object_name.tar.gz ($object_size) downloaded."
+
+            tar -xvf "$object_name.tar.gz"
+
+            popd >/dev/null
+        fi
+
+    fi
+
+    local do_open=$(abcli_option_int "$open_options" open 0)
+    [[ "$do_open" == 1 ]] &&
+        abcli_open filename=$filename,$open_options \
+            $object_name
+
+    return 0
+}

bluer_objects/.abcli/file.sh

@@ -0,0 +1,8 @@
+#! /usr/bin/env bash
+
+function bluer_objects_file() {
+    python3 -m bluer_objects.file \
+        "$1" \
+        --filename "$2" \
+        "${@:3}"
+}

bluer_objects/.abcli/gif.sh

@@ -0,0 +1,27 @@
+#! /usr/bin/env bash
+
+function abcli_gif() {
+    local options=$1
+    local do_dryrun=$(abcli_option_int "$options" dryrun 0)
+    local do_download=$(abcli_option_int "$options" download $(abcli_not $do_dryrun))
+    local do_upload=$(abcli_option_int "$options" upload $(abcli_not $do_dryrun))
+
+    local object_name=$(abcli_clarify_object $2 .)
+
+    [[ "$do_download" == 1 ]] &&
+        abcli_download - $object_name
+
+    abcli_log "generating animated gif: $object_name ..."
+
+    abcli_eval dryrun=$do_dryrun \
+        python3 -m bluer_objects.graphics \
+        generate_animated_gif \
+        --object_name $object_name \
+        "${@:3}"
+    local status="$?"
+
+    [[ "$do_upload" == 1 ]] &&
+        abcli_upload - $object_name
+
+    return $status
+}

bluer_objects/.abcli/host.sh

@@ -0,0 +1,29 @@
+#! /usr/bin/env bash
+
+function abcli_host() {
+    local task=$(abcli_unpack_keyword $1 void)
+    local options=$2
+
+    if [ $task == "get" ]; then
+        python3 -m blue_options.host \
+            get \
+            --keyword "$2" \
+            "${@:3}"
+        return
+    fi
+
+    if [ $task == "reboot" ]; then
+        abcli_eval ,$options \
+            sudo reboot
+        return
+    fi
+
+    if [ $task == "shutdown" ]; then
+        abcli_eval ,$options \
+            sudo shutdown -h now
+        return
+    fi
+
+    abcli_log_error "@host: $task: command not found."
+    return 1
+}

bluer_objects/.abcli/ls.sh

@@ -0,0 +1,24 @@
+#! /usr/bin/env bash
+
+function abcli_ls() {
+    local options=$1
+    local on_cloud=$(abcli_option_int "$options" cloud 0)
+    local on_local=$(abcli_option_int "$options" local 0)
+
+    [[ "$on_cloud" == 1 ]] ||
+        [[ "$on_local" == 1 ]] &&
+        local object_name=$(abcli_clarify_object $2 .)
+
+    if [ "$on_cloud" == 1 ]; then
+        abcli_eval - \
+            aws s3 ls $ABCLI_S3_OBJECT_PREFIX/$object_name/
+    elif [ "$on_local" == 1 ]; then
+        abcli_eval - \
+            ls -1lh $ABCLI_OBJECT_ROOT/$object_name
+    else
+        abcli_eval - \
+            ls -1 "$@"
+    fi
+
+    return 0
+}

bluer_objects/.abcli/metadata/get.sh

@@ -0,0 +1,24 @@
+#! /usr/bin/env bash
+
+function abcli_metadata_get() {
+    local options=$1
+    local source_type=$(abcli_option_choice "$options" object,path,filename object)
+
+    local source=$2
+    [[ "$source_type" == object ]] &&
+        source=$(abcli_clarify_object $2 .)
+
+    local key=$(abcli_option "$options" key)
+    local default=$(abcli_option "$options" default)
+
+    python3 -m bluer_objects.metadata get \
+        --default "$default" \
+        --delim $(abcli_option "$options" delim ,) \
+        --dict_keys $(abcli_option_int "$options" dict.keys 0) \
+        --dict_values $(abcli_option_int "$options" dict.values 0) \
+        --filename $(abcli_option "$options" filename metadata.yaml) \
+        --key "$key" \
+        --source "$source" \
+        --source_type $source_type \
+        "${@:3}"
+}

bluer_objects/.abcli/metadata/post.sh

@@ -0,0 +1,22 @@
+#! /usr/bin/env bash
+
+function abcli_metadata_post() {
+    local key=$1
+
+    local value=$2
+
+    local options=$3
+    local source_type=$(abcli_option_choice "$options" object,path,filename object)
+
+    local source=$4
+    [[ "$source_type" == object ]] &&
+        source=$(abcli_clarify_object $4 .)
+
+    python3 -m bluer_objects.metadata post \
+        --filename $(abcli_option "$options" filename metadata.yaml) \
+        --key "$key" \
+        --value "$value" \
+        --source "$source" \
+        --source_type $source_type \
+        "${@:5}"
+}

bluer_objects/.abcli/metadata.sh

@@ -0,0 +1,16 @@
+#! /usr/bin/env bash
+
+function abcli_metadata() {
+    local task=${1:-help}
+
+    local function_name="abcli_metadata_$task"
+    if [[ $(type -t $function_name) == "function" ]]; then
+        $function_name "${@:2}"
+        return
+    fi
+
+    abcli_log_error "@metadata: $task: command not found."
+    return 1
+}
+
+abcli_source_caller_suffix_path /metadata

bluer_objects/.abcli/mlflow/browse.sh

@@ -0,0 +1,36 @@
+#! /usr/bin/env bash
+
+function abcli_mlflow_browse() {
+    local options=$1
+    local browse_experiment=$(abcli_option_int "$options" experiment 0)
+
+    local url=$DATABRICKS_HOST/$ABCLI_MLFLOW_URL_SUBDOMAIN
+
+    if [ $(abcli_option_int "$options" databricks 0) == 1 ]; then
+        url="https://accounts.cloud.databricks.com/"
+    elif [ $(abcli_option_int "$options" host 0) == 1 ]; then
+        : # do nothing
+    elif [ $(abcli_option_int "$options" models 0) == 1 ]; then
+        url="$url/models"
+    else
+        local object_name=$(abcli_clarify_object $2 .)
+
+        local experiment_id=$(abcli_mlflow get_id $object_name)
+        if [ -z "$experiment_id" ]; then
+            abcli_log_error "@mlflow: browse: $object_name: object not found."
+            return 1
+        fi
+        abcli_log "experiment id: $experiment_id"
+
+        url="$url/experiments/$experiment_id"
+
+        if [[ "$browse_experiment" == 0 ]]; then
+            local last_run_id=$(abcli_mlflow get_run_id $object_name --count 1)
+            abcli_log "last run id: $last_run_id"
+
+            url="$url/runs/$last_run_id"
+        fi
+    fi
+
+    abcli_browse $url
+}

bluer_objects/.abcli/mlflow/cache.sh

@@ -0,0 +1,31 @@
+#! /usr/bin/env bash
+
+function abcli_mlflow_cache() {
+    local task=$(abcli_unpack_keyword $1 help)
+
+    local keyword=$2
+
+    if [ "$task" == "read" ]; then
+
+        abcli_mlflow_tags get \
+            $keyword \
+            --tag referent \
+            "${@:3}"
+
+        return
+    fi
+
+    if [ "$task" == "write" ]; then
+        local value=$3
+
+        abcli_mlflow_tags set \
+            $keyword \
+            referent=$value \
+            "${@:4}"
+
+        return
+    fi
+
+    abcli_log_error "@mlflow: cache: $task: command not found."
+    return 1
+}

bluer_objects/.abcli/mlflow/list_registered_models.sh

@@ -0,0 +1,9 @@
+#! /usr/bin/env bash
+
+function abcli_mlflow_list_registered_models() {
+    local options=$1
+
+    python3 -m bluer_objects.mlflow \
+        list_registered_models \
+        "${@:2}"
+}

bluer_objects/.abcli/mlflow/log_artifacts.sh

@@ -0,0 +1,10 @@
+#! /usr/bin/env bash
+
+function abcli_mlflow_log_artifacts() {
+    local object_name=$(abcli_clarify_object $1 .)
+
+    python3 -m bluer_objects.mlflow \
+        log_artifacts \
+        --object_name $object_name \
+        "${@:2}"
+}

bluer_objects/.abcli/mlflow/log_run.sh

@@ -0,0 +1,10 @@
+#! /usr/bin/env bash
+
+function abcli_mlflow_log_run() {
+    local object_name=$(abcli_clarify_object $1 .)
+
+    python3 -m bluer_objects.mlflow \
+        log_run \
+        --object_name $object_name \
+        "${@:2}"
+}

bluer_objects/.abcli/mlflow/run.sh

@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+function abcli_mlflow_run() {
+    local object_name=$(abcli_clarify_object $1 .)
+
+    python3 -m bluer_objects.mlflow \
+        start_end_run \
+        --object_name $object_name \
+        --start_end "$2" \
+        "${@:3}"
+}

bluer_objects/.abcli/mlflow/tags/clone.sh

@@ -0,0 +1,15 @@
+#! /usr/bin/env bash
+
+function abcli_mlflow_tags_clone() {
+    local source_object=$(abcli_clarify_object $1 ..)
+
+    local destination_object=$(abcli_clarify_object $2 .)
+
+    abcli_log "mlflow: tags: clone: $source_object -> $destination_object ..."
+
+    python3 -m bluer_objects.mlflow \
+        clone_tags \
+        --destination_object $destination_object \
+        --source_object $source_object \
+        "${@:3}"
+}

bluer_objects/.abcli/mlflow/tags/get.sh

@@ -0,0 +1,10 @@
+#! /usr/bin/env bash
+
+function abcli_mlflow_tags_get() {
+    local object_name=$(abcli_clarify_object $1 .)
+
+    python3 -m bluer_objects.mlflow \
+        get_tags \
+        --object_name $object_name \
+        "${@:2}"
+}

bluer_objects/.abcli/mlflow/tags/search.sh

@@ -0,0 +1,12 @@
+#! /usr/bin/env bash
+
+function abcli_mlflow_tags_search() {
+    local options=$1
+    local is_explicit=$(abcli_option_int "$options" explicit 0)
+
+    python3 -m bluer_objects.mlflow \
+        search \
+        --explicit_query $is_explicit \
+        --tags "$options" \
+        "${@:2}"
+}

bluer_objects/.abcli/mlflow/tags/set.sh

@@ -0,0 +1,13 @@
+#! /usr/bin/env bash
+
+function abcli_mlflow_tags_set() {
+    local object_name=$(abcli_clarify_object $1 .)
+
+    local tags=$2
+
+    python3 -m bluer_objects.mlflow \
+        set_tags \
+        --object_name $object_name \
+        --tags "$tags" \
+        "${@:3}"
+}

bluer_objects/.abcli/mlflow/tags.sh

@@ -0,0 +1,16 @@
+#! /usr/bin/env bash
+
+function abcli_mlflow_tags() {
+    local task=$1
+
+    local function_name=abcli_mlflow_tags_$task
+    if [[ $(type -t $function_name) == "function" ]]; then
+        $function_name "${@:2}"
+        return
+    fi
+
+    abcli_log_error "@mlflow: tags: $task: command not found."
+    return 1
+}
+
+abcli_source_caller_suffix_path /tags

bluer_objects/.abcli/mlflow/test.sh

@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+function abcli_mlflow_test() {
+    local options=$1
+    local do_dryrun=$(abcli_option_int "$options" dryrun 0)
+
+    abcli_eval dryrun=$do_dryrun \
+        python3 -m bluer_objects.mlflow \
+        test \
+        "$@"
+}

bluer_objects/.abcli/mlflow/transition.sh

@@ -0,0 +1,20 @@
+#! /usr/bin/env bash
+
+export ABCLI_MLFLOW_STAGES="Staging|Production|Archived"
+
+function abcli_mlflow_transition() {
+    local options=$1
+    local model_name=$(abcli_option "$options" model)
+    local stage_name=$(abcli_option_choice "$options" $(echo $ABCLI_MLFLOW_STAGES | tr \| ,) Staging)
+    local version=$(abcli_option "$options" version)
+
+    local description="$2"
+
+    python3 -m bluer_objects.mlflow \
+        transition \
+        --model_name "$model_name" \
+        --version "$version" \
+        --stage_name "$stage_name" \
+        --description "$description" \
+        "${@:3}"
+}

bluer_objects/.abcli/mlflow.sh

@@ -0,0 +1,29 @@
+#! /usr/bin/env bash
+
+export MLFLOW_TRACKING_URI="databricks"
+
+function abcli_mlflow() {
+    local task=$(abcli_unpack_keyword $1 help)
+
+    local function_name=abcli_mlflow_$task
+    if [[ $(type -t $function_name) == "function" ]]; then
+        $function_name "${@:2}"
+        return
+    fi
+
+    if [[ ",get_id,get_run_id,rm," == *",$task,"* ]]; then
+        local object_name=$(abcli_clarify_object $2 .)
+
+        python3 -m bluer_objects.mlflow \
+            $task \
+            --object_name $object_name \
+            "${@:3}"
+
+        return
+    fi
+
+    abcli_log_error "@mlflow: $task: command not found."
+    return 1
+}
+
+abcli_source_caller_suffix_path /mlflow