bluer-objects 6.3.1__py3-none-any.whl

bluer_objects/mysql/tags/__main__.py

@@ -0,0 +1,130 @@
+import argparse
+
+from blueness import module
+from blueness.argparse.generic import sys_exit
+
+from bluer_objects import NAME
+from bluer_objects.mysql.tags.functions import clone, create, get, search, set_
+from bluer_objects.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+
+parser = argparse.ArgumentParser(NAME)
+parser.add_argument(
+    "task",
+    type=str,
+    default="get",
+    help="clone|create|get|search|set",
+)
+parser.add_argument(
+    "--after",
+    type=str,
+    default="",
+    help="123-4-e",
+)
+parser.add_argument(
+    "--before",
+    type=str,
+    default="",
+)
+parser.add_argument(
+    "--count",
+    type=int,
+    default=-1,
+)
+parser.add_argument(
+    "--offset",
+    type=int,
+    default=0,
+)
+parser.add_argument(
+    "--delim",
+    type=str,
+    default=", ",
+)
+parser.add_argument(
+    "--host",
+    default=-1,
+    type=int,
+    help="0|1|-1",
+)
+parser.add_argument(
+    "--item_name",
+    default="object",
+    type=str,
+)
+parser.add_argument(
+    "--log",
+    default=1,
+    type=int,
+    help="0|1",
+)
+parser.add_argument(
+    "--object",
+    type=str,
+    default="",
+)
+parser.add_argument(
+    "--object_2",
+    type=str,
+    default="",
+)
+parser.add_argument(
+    "--shuffle",
+    default=0,
+    type=int,
+    help="0|1",
+)
+parser.add_argument(
+    "--tag",
+    type=str,
+    default="",
+)
+parser.add_argument(
+    "--tags",
+    type=str,
+    default="",
+    help="tag_1,~tag_2",
+)
+parser.add_argument(
+    "--type",
+    type=str,
+    default="",
+)
+args = parser.parse_args()
+
+delim = " " if args.delim == "space" else args.delim
+
+success = False
+output = None
+if args.task == "clone":
+    success = clone(args.object, args.object_2)
+elif args.task == "create":
+    success = create()
+elif args.task == "get":
+    output = get(args.object)
+    success = True
+elif args.task == "search":
+    output = search(
+        args.tags,
+        after=args.after,
+        before=args.before,
+        count=args.count,
+        host=args.host,
+        shuffle=args.shuffle,
+        offset=args.offset,
+    )
+    success = True
+elif args.task == "set":
+    success = set_(args.object, args.tags)
+else:
+    success = None
+
+if success is True and output is not None:
+    if args.log:
+        logger.info(f"{len(output):,} {args.item_name}(s): {delim.join(output)}")
+    else:
+        print(delim.join(output))
+
+sys_exit(logger, NAME, args.task, success, log=args.log)

bluer_objects/mysql/tags/functions.py

@@ -0,0 +1,203 @@
+from typing import List
+from functools import reduce
+import random
+import re
+
+from blue_options.options import Options
+
+from bluer_objects.mysql.table import Table
+from bluer_objects.logger import logger
+
+
+def clone(
+    object_1: str,
+    object_2: str,
+) -> bool:
+    return set_(object_2, get(object_1))
+
+
+def create() -> bool:
+    return Table.Create(
+        "tags",
+        [
+            "keyword VARCHAR(256) NOT NULL",
+            "tag VARCHAR(4096) NOT NULL",
+            "value BIT NOT NULL",
+        ],
+    )
+
+
+def get(keyword: str) -> List[str]:
+    table = Table(name="tags")
+
+    if not table.connect():
+        return []
+
+    success, output = table.execute(
+        "SELECT t.tag,t.value "
+        f"FROM {table.name} t "
+        "INNER JOIN ( "
+        "SELECT tag, MAX(timestamp) AS max_timestamp "
+        f"FROM {table.name} "
+        f'WHERE keyword="{keyword}" GROUP BY tag '
+        ") tm "
+        "ON t.tag=tm.tag AND t.timestamp=tm.max_timestamp "
+        f'WHERE keyword="{keyword}";',
+    )
+
+    if success:
+        success = table.disconnect()
+
+    if not success:
+        return []
+
+    return sorted([thing[0] for thing in output if thing[1] == b"\x01"])
+
+
+def search(
+    tags: List[str],
+    after: str = "",
+    before: str = "",
+    count: int = -1,
+    host: int = -1,  # limit to/exclude/ignore (1/0/-1) hosts.
+    return_timestamp: bool = False,
+    shuffle: bool = False,
+    offset: int = 0,
+) -> List[str]:
+    if isinstance(tags, str):
+        tags = tags.split(",")
+
+    included_tags = []
+    excluded_tags = []
+    for tag in tags:
+        if tag:
+            if tag[0] in "~-!":
+                excluded_tags += [tag[1:]]
+            else:
+                included_tags += [tag]
+
+    table = Table(name="tags")
+
+    table.connect()
+
+    list_of_keywords = None
+    timestamp = {}
+    for tag in included_tags:
+        success, output = table.execute(
+            "SELECT t.keyword,t.value,t.timestamp "
+            "FROM abcli.tags t "
+            "INNER JOIN ( "
+            "SELECT keyword, MAX(timestamp) AS max_timestamp "
+            "FROM abcli.tags "
+            f'WHERE tag="{tag}" GROUP BY keyword '
+            ") tm "
+            "ON t.keyword=tm.keyword AND t.timestamp=tm.max_timestamp "
+            f'WHERE tag="{tag}"; '
+        )
+        if not success:
+            list_of_keywords = []
+            break
+
+        list_of_keywords_ = [thing[0] for thing in output if thing[1] == b"\x01"]
+
+        if return_timestamp:
+            for thing in output:
+                if thing[1] == b"\x01":
+                    timestamp[thing[0]] = thing[2]
+
+        list_of_keywords = (
+            list_of_keywords_
+            if list_of_keywords is None
+            else [
+                keyword for keyword in list_of_keywords if keyword in list_of_keywords_
+            ]
+        )
+
+    table.disconnect()
+
+    list_of_keywords = [] if list_of_keywords is None else sorted(list_of_keywords)
+
+    if after:
+        list_of_keywords = [keyword for keyword in list_of_keywords if keyword >= after]
+
+    if before:
+        list_of_keywords = [
+            keyword for keyword in list_of_keywords if keyword <= before
+        ]
+
+    excluded_keywords = reduce(
+        lambda x, y: x + y,
+        [
+            search(
+                tag,
+                after=after,
+                before=before,
+                count=-1,
+                host=host,
+            )
+            for tag in excluded_tags
+        ],
+        [],
+    )
+
+    list_of_keywords = [
+        keyword for keyword in list_of_keywords if keyword not in excluded_keywords
+    ]
+
+    if shuffle:
+        random.shuffle(list_of_keywords)
+    else:
+        list_of_keywords = list_of_keywords[::-1]
+
+    p = re.compile("([0-9]{13}|(0|1)[0-9,a-z]{15}|i-[0-9,a-z]{17})")
+    if host == 1:
+        list_of_keywords = [keyword for keyword in list_of_keywords if p.match(keyword)]
+    if host == 0:
+        list_of_keywords = [
+            keyword for keyword in list_of_keywords if not p.match(keyword)
+        ]
+
+    list_of_keywords = list_of_keywords[offset:]
+
+    list_of_keywords = (
+        list_of_keywords[:count]
+        if count > 0
+        else [] if count != -1 else list_of_keywords
+    )
+
+    return (list_of_keywords, timestamp) if return_timestamp else list_of_keywords
+
+
+def set_(
+    keyword: str,
+    tags: List[str],
+) -> bool:
+    table = Table(name="tags")
+
+    if isinstance(tags, list):
+        tags = ",".join(tags)
+    if isinstance(tags, str):
+        tags = Options(tags)
+
+    if not table.connect():
+        return False
+
+    tags = {tag.strip(): value for tag, value in tags.items()}
+
+    success = True
+    for tag in tags:
+        if not table.insert(
+            "keyword,tag,value".split(","),
+            [keyword, tag, 1 if tags[tag] else 0],
+        ):
+            success = False
+        else:
+            if tags[tag]:
+                logger.info(f"{keyword} += #{tag}.")
+            else:
+                logger.info(f"{keyword} -= #{tag}.")
+
+    if not table.disconnect():
+        return False
+
+    return success

bluer_objects/objects.py

@@ -0,0 +1,167 @@
+import os
+
+from blue_options import string
+
+from bluer_objects import file, path
+from bluer_objects.env import (
+    ABCLI_OBJECT_ROOT,
+    abcli_object_name,
+    ABCLI_S3_OBJECT_PREFIX,
+)
+from bluer_objects.host import shell
+from bluer_objects.logger import logger
+
+
+def download(
+    object_name: str,
+    filename: str = "",
+    overwrite: bool = False,
+) -> bool:
+    if not ABCLI_S3_OBJECT_PREFIX:
+        logger.error("ABCLI_S3_OBJECT_PREFIX is not set.")
+        return False
+
+    if not object_name:
+        logger.error("object_name not found.")
+        return False
+
+    if (
+        filename
+        and not overwrite
+        and file.exists(
+            path_of(
+                object_name=object_name,
+                filename=filename,
+            )
+        )
+    ):
+        return True
+
+    return (
+        shell(
+            "aws s3 cp {}/{}/{} {}".format(
+                ABCLI_S3_OBJECT_PREFIX,
+                object_name,
+                filename,
+                object_path(object_name, create=True),
+            )
+        )
+        if filename
+        else shell(
+            "aws s3 sync {}/{}/ {}".format(
+                ABCLI_S3_OBJECT_PREFIX,
+                object_name,
+                object_path(object_name, create=True),
+            )
+        )
+    )
+
+
+def list_of_files(
+    object_name: str,
+    cloud: bool = False,
+    **kwargs,
+):
+    if cloud:
+        from bluer_objects.storage import instance as storage
+
+        return storage.list_of_objects(
+            object_name,
+            **kwargs,
+        )
+
+    return file.list_of(
+        os.path.join(
+            ABCLI_OBJECT_ROOT,
+            object_name,
+            "*",
+        ),
+        **kwargs,
+    )
+
+
+def object_path(
+    object_name=".",
+    create=False,
+):
+    output = os.path.join(
+        ABCLI_OBJECT_ROOT,
+        abcli_object_name if object_name == "." else object_name,
+    )
+
+    if create:
+        os.makedirs(output, exist_ok=True)
+
+    return output
+
+
+def path_of(
+    filename,
+    object_name=".",
+    create=False,
+):
+    return os.path.join(
+        object_path(object_name, create),
+        filename,
+    )
+
+
+def signature(info=None, object_name="."):
+    return [
+        "{}{}".format(
+            abcli_object_name if object_name == "." else object_name,
+            "" if info is None else f"/{str(info)}",
+        ),
+        string.pretty_date(include_time=False),
+        string.pretty_date(include_date=False, include_zone=True),
+    ]
+
+
+def unique_object(
+    prefix: str = "",
+    include_time: bool = True,
+):
+    object_name = string.pretty_date(
+        as_filename=True,
+        include_time=include_time,
+        unique=True,
+    )
+    if prefix:
+        object_name = f"{prefix}-{object_name}"
+
+    path.create(object_path(object_name))
+
+    logger.info(f"📂 {object_name}")
+
+    return object_name
+
+
+def upload(
+    object_name: str,
+    filename: str = "",
+) -> bool:
+    if not ABCLI_S3_OBJECT_PREFIX:
+        logger.error("ABCLI_S3_OBJECT_PREFIX is not set.")
+        return False
+
+    if not object_name:
+        logger.error("object_name not found.")
+        return False
+
+    return (
+        shell(
+            "aws s3 cp {} {}/{}/".format(
+                path_of(filename=filename, object_name=object_name),
+                ABCLI_S3_OBJECT_PREFIX,
+                object_name,
+            )
+        )
+        if filename
+        else shell(
+            "aws s3 sync {} {}/{}/".format(
+                object_path(object_name, create=True),
+                ABCLI_S3_OBJECT_PREFIX,
+                object_name,
+            )
+        )
+    )

bluer_objects/path.py

@@ -0,0 +1,194 @@
+import os
+from typing import Any, Union, List
+import pathlib
+import shutil
+
+from blueness import module
+from blue_options import string
+from blue_options.logger import crash_report
+
+from bluer_objects import NAME
+from bluer_objects.logger import logger
+from bluer_objects.env import abcli_object_path
+
+
+NAME = module.name(__file__, NAME)
+
+
+def absolute(
+    path: str,
+    reference: Union[None, str] = None,
+):
+    if reference is None:
+        reference = current()
+    assert isinstance(reference, str)
+
+    path = path.replace("/", os.sep)
+    reference = reference.replace("/", os.sep)
+
+    return (
+        reference
+        if not path
+        else (
+            path
+            if path[0] != "."
+            else str(pathlib.Path(os.path.join(reference, path)).resolve())
+        )
+    )
+
+
+def auxiliary(
+    nickname: str,
+    add_timestamp: bool = True,
+):
+    path = os.path.join(
+        abcli_object_path,
+        "auxiliary",
+        "-".join(
+            [nickname]
+            + (
+                [
+                    string.pretty_date(
+                        as_filename=True,
+                        squeeze=True,
+                        unique=True,
+                    )
+                ]
+                if add_timestamp
+                else []
+            )
+        ),
+    )
+
+    assert create(path)
+
+    return path
+
+
+def copy(
+    source: str,
+    destination: str,
+) -> bool:
+    if not create(parent(destination)):
+        return False
+
+    try:
+        shutil.copytree(source, destination)
+        return True
+    except:
+        crash_report(f"{NAME}: copy({source},{destination}): failed.")
+        return False
+
+
+def create(
+    path: str,
+    log: bool = False,
+) -> bool:
+    if not path or exists(path):
+        return True
+
+    try:
+        os.makedirs(path)
+    except:
+        crash_report(f"{NAME}: create({path}): failed.")
+        return False
+
+    if log:
+        logger.info(f"{NAME}.create({path})")
+
+    return True
+
+
+def current() -> str:
+    return os.getcwd()
+
+
+def delete(
+    path: str,
+) -> bool:
+    try:
+        # https://docs.python.org/3/library/shutil.html#shutil.rmtree
+        shutil.rmtree(path)
+        return True
+    except:
+        crash_report(f"{NAME}: delete({path}): failed.")
+        return False
+
+
+def exists(path: str) -> bool:
+    return os.path.exists(path) and os.path.isdir(path)
+
+
+def list_of(
+    path: str,
+    recursive: bool = False,
+) -> List[str]:
+    if not exists(path):
+        return []
+
+    # http://stackabuse.com/python-list-files-in-a-directory/
+    output = []
+    try:
+        for entry in os.scandir(path):
+            if entry.is_file():
+                continue
+
+            path_name = os.path.join(path, entry.name)
+
+            output.append(path_name)
+
+            if recursive:
+                output += list_of(path_name, recursive=recursive)
+    except:
+        crash_report(f"-{NAME}: list_of({path}): failed.")
+
+    return output
+
+
+def move(
+    source: str,
+    destination: str,
+) -> bool:
+    if not create(parent(destination)):
+        return False
+
+    try:
+        shutil.move(source, destination)
+        return True
+    except:
+        crash_report(f"{NAME}: move({source},{destination}): failed.")
+        return False
+
+
+def name(path: str) -> str:
+    if not path:
+        return path
+
+    if path[-1] == os.sep:
+        path = path[:-1]
+
+    path_components = path.split(os.sep)
+    return "" if not path_components else path_components[-1]
+
+
+def parent(
+    path: str,
+    depth: int = 1,
+) -> str:
+    # Add os.sep at the end of path, if it already does not exist.
+    if path:
+        if path[-1] != os.sep:
+            path = path + os.sep
+
+    return os.sep.join(path.split(os.sep)[: -depth - 1]) + os.sep
+
+
+def relative(
+    path: str,
+    reference: Union[Any, str] = None,
+):
+    # https://stackoverflow.com/a/918174
+    return os.path.relpath(
+        path,
+        current() if reference is None else reference,
+    )

bluer_objects/sample.env

@@ -0,0 +1,16 @@
+# to complete the `rds` section create a database as follows:
+# go to `AWS Management Console` -> `RDS` -> `Create database`.
+# use `Standard Create` -> `MySQL` -> `MySQL 8.0.20` -> `Dev/Test`.
+# set `DB instance identifier` to `abcli`.
+# set `Credentials Settings` -> `Master username` and `Master Password` and update `host`, `username`, and `password` here.
+# set `DB instance size` to `Burstable Classes` and select `db.t3.small`.
+# set `Storage` to `20` and disable `Storage autoscaling`.
+# in `Connectivity` set `Public Access` to `Yes`.
+# in `Additional Configuration` set `initial database name` to `abcli`.
+ABCLI_AWS_RDS_HOST=
+ABCLI_AWS_RDS_PASSWORD=
+
+# https://docs.databricks.com/en/mlflow/tracking.html
+# https://docs.databricks.com/en/dev-tools/auth/pat.html#databricks-personal-access-tokens-for-workspace-users
+DATABRICKS_HOST=
+DATABRICKS_TOKEN=

bluer_objects/storage/__init__.py

@@ -0,0 +1,3 @@
+from bluer_objects.storage.classes import Storage
+
+instance = Storage()