bluer-objects 6.3.1__py3-none-any.whl

bluer_objects/metadata/__main__.py

@@ -0,0 +1,110 @@
+import argparse
+
+from blueness import module
+from blueness.argparse.generic import sys_exit
+
+
+from bluer_objects import NAME
+from bluer_objects.metadata.get import get
+from bluer_objects.metadata.post import post
+from bluer_objects.metadata.enums import MetadataSourceType
+from bluer_objects.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+
+parser = argparse.ArgumentParser(NAME)
+parser.add_argument(
+    "task",
+    type=str,
+    default="",
+    help="get|post",
+)
+parser.add_argument(
+    "--default",
+    type=str,
+    default="",
+)
+parser.add_argument(
+    "--delim",
+    type=str,
+    default="",
+)
+parser.add_argument(
+    "--dict_keys",
+    type=int,
+    default=0,
+    help="0|1",
+)
+parser.add_argument(
+    "--dict_values",
+    type=int,
+    default=0,
+    help="0|1",
+)
+parser.add_argument(
+    "--filename",
+    type=str,
+    default="metadata.yaml",
+)
+parser.add_argument(
+    "--is_base64_encoded",
+    type=int,
+    default=0,
+    help="0|1",
+)
+parser.add_argument(
+    "--key",
+    type=str,
+)
+parser.add_argument(
+    "--source",
+    type=str,
+)
+parser.add_argument(
+    "--source_type",
+    type=str,
+    default=MetadataSourceType.FILENAME.name.lower(),
+    help="|".join([source_type.name.lower() for source_type in MetadataSourceType]),
+)
+parser.add_argument(
+    "--value",
+    type=str,
+)
+parser.add_argument(
+    "--verbose",
+    type=int,
+    default="0",
+    help="0|1",
+)
+args = parser.parse_args()
+
+delim = " " if args.delim == "space" else args.delim
+
+success = False
+if args.task == "get":
+    success = True
+    output = get(
+        key=args.key,
+        default=args.default,
+        source=args.source,
+        source_type=MetadataSourceType[args.source_type.upper()],
+        dict_keys=args.dict_keys,
+        dict_values=args.dict_values,
+        filename=args.filename,
+    )
+    print(delim.join(output) if isinstance(output, list) else output)
+elif args.task == "post":
+    success = post(
+        key=args.key,
+        value=args.value,
+        filename=args.filename,
+        source=args.source,
+        source_type=MetadataSourceType[args.source_type.upper()],
+        is_base64_encoded=args.is_base64_encoded,
+        verbose=args.verbose == 1,
+    )
+else:
+    success = None
+
+sys_exit(logger, NAME, args.task, success)

bluer_objects/metadata/enums.py

@@ -0,0 +1,29 @@
+import os
+from enum import Enum, auto
+
+from bluer_objects import objects
+from bluer_objects.env import abcli_object_path
+
+
+class MetadataSourceType(Enum):
+    FILENAME = auto()
+    OBJECT = auto()
+    PATH = auto()
+
+    def filename(self, source: str, filename="metadata.yaml"):
+        if self == MetadataSourceType.FILENAME:
+            return source
+
+        if self == MetadataSourceType.OBJECT:
+            return os.path.join(
+                objects.object_path(source),
+                filename,
+            )
+
+        if self == MetadataSourceType.PATH:
+            return os.path.join(
+                abcli_object_path if source == "." else source,
+                filename,
+            )
+
+        return f"unknown-source-type:{self.name.lower()}"

bluer_objects/metadata/get.py

@@ -0,0 +1,89 @@
+from typing import Any
+
+
+from bluer_objects import file
+from bluer_objects import objects
+from bluer_objects.metadata.enums import MetadataSourceType
+
+
+def get(
+    key,
+    default="",
+    source=".",
+    source_type: MetadataSourceType = MetadataSourceType.FILENAME,
+    filename: str = "metadata.yaml",
+    dict_keys: bool = False,
+    dict_values: bool = False,
+) -> Any:
+    success, metadata = file.load_yaml(
+        source_type.filename(source, filename),
+        ignore_error=True,
+    )
+    if not success:
+        return default
+
+    try:
+        output = metadata
+        for key_ in [key_ for key_ in key.split(".") if key_]:
+            output = output.get(key_, default)
+
+        if dict_keys:
+            output = list(output.keys())
+        elif dict_values:
+            output = list(output.values())
+
+        return output
+    except Exception as e:
+        return type(e).__name__
+
+
+def get_from_file(
+    filename: str,
+    key,
+    default="",
+    **kwargs,
+) -> Any:
+    return get(
+        key=key,
+        default=default,
+        source=filename,
+        source_type=MetadataSourceType.FILENAME,
+        **kwargs,
+    )
+
+
+def get_from_object(
+    object_name: str,
+    key,
+    default="",
+    download: bool = False,
+    **kwargs,
+) -> Any:
+    if download and not objects.download(
+        object_name=object_name,
+        filename="metadata.yaml",
+    ):
+        return default
+
+    return get(
+        key=key,
+        default=default,
+        source=object_name,
+        source_type=MetadataSourceType.OBJECT,
+        **kwargs,
+    )
+
+
+def get_from_path(
+    path: str,
+    key: str,
+    default="",
+    **kwargs,
+) -> Any:
+    return get(
+        key=key,
+        default=default,
+        source=path,
+        source_type=MetadataSourceType.PATH,
+        **kwargs,
+    )

bluer_objects/metadata/post.py

@@ -0,0 +1,101 @@
+from typing import Any
+import base64
+import copy
+
+from blueness import module
+
+from bluer_objects.metadata.enums import MetadataSourceType
+from bluer_objects import NAME, file
+from bluer_objects import objects
+from bluer_objects.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+
+def post(
+    key: str,
+    value: Any,
+    filename: str = "metadata.yaml",
+    source=".",
+    source_type: MetadataSourceType = MetadataSourceType.FILENAME,
+    is_base64_encoded=False,
+    verbose: bool = False,
+) -> bool:
+    if is_base64_encoded:
+        value = str(base64.b64decode(value))
+
+    filename = source_type.filename(source, filename)
+
+    _, metadata = file.load_yaml(filename, ignore_error=True)
+
+    metadata[key] = copy.deepcopy(value)
+
+    logger.info(
+        "{}.post[{}]: {}{}".format(
+            NAME,
+            filename,
+            key,
+            f"={value}" if verbose else "",
+        )
+    )
+
+    return file.save_yaml(filename, metadata)
+
+
+def post_to_file(
+    filename: str,
+    key: str,
+    value: Any,
+    **kwargs,
+) -> bool:
+    return post(
+        key=key,
+        value=value,
+        source=filename,
+        source_type=MetadataSourceType.FILENAME,
+        **kwargs,
+    )
+
+
+def post_to_object(
+    object_name: str,
+    key: str,
+    value: Any,
+    download: bool = False,
+    upload: bool = False,
+    **kwargs,
+) -> bool:
+    if download and not objects.download(
+        object_name=object_name,
+        filename="metadata.yaml",
+    ):
+        return False
+
+    if not post(
+        key=key,
+        value=value,
+        source=object_name,
+        source_type=MetadataSourceType.OBJECT,
+        **kwargs,
+    ):
+        return False
+
+    return not upload or objects.upload(
+        object_name=object_name,
+        filename="metadata.yaml",
+    )
+
+
+def post_to_path(
+    path: str,
+    key: str,
+    value: Any,
+    **kwargs,
+) -> bool:
+    return post(
+        key=key,
+        value=value,
+        source=path,
+        source_type=MetadataSourceType.PATH,
+        **kwargs,
+    )

bluer_objects/mlflow/__init__.py

@@ -0,0 +1,28 @@
+from bluer_objects.mlflow.logging import (
+    log_artifacts,
+    log_run,
+)
+from bluer_objects.mlflow.models import (
+    list_registered_models,
+    transition,
+)
+from bluer_objects.mlflow.objects import (
+    get_id,
+    to_experiment_name,
+    to_object_name,
+    rm,
+)
+from bluer_objects.mlflow.runs import (
+    end_run,
+    get_run_id,
+    start_run,
+)
+from bluer_objects.mlflow.tags import (
+    create_filter_string,
+    get_tags,
+    search,
+    set_tags,
+)
+from bluer_objects.mlflow.testing import (
+    test,
+)

bluer_objects/mlflow/__main__.py

@@ -0,0 +1,271 @@
+import argparse
+from functools import reduce
+import os
+
+from blueness.argparse.generic import sys_exit
+from blueness import module
+
+from bluer_objects import NAME
+from bluer_objects.mlflow.logging import (
+    log_artifacts,
+    log_run,
+)
+from bluer_objects.mlflow.models import (
+    list_registered_models,
+    transition,
+)
+from bluer_objects.mlflow.objects import (
+    get_id,
+    rm,
+)
+from bluer_objects.mlflow.runs import (
+    end_run,
+    get_run_id,
+    start_run,
+)
+from bluer_objects.mlflow.tags import (
+    create_filter_string,
+    get_tags,
+    search,
+    set_tags,
+)
+from bluer_objects.mlflow.testing import (
+    test,
+)
+from bluer_objects.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+
+parser = argparse.ArgumentParser(NAME)
+parser.add_argument(
+    "task",
+    type=str,
+    default="",
+    help="clone_tags|create_filter_string|get_id|get_run_id|get_tags|list_registered_models|log_artifacts|log_run|rm|search|set_tags|start_end_run|test|transition",
+)
+parser.add_argument(
+    "--count",
+    type=int,
+    default=-1,
+)
+parser.add_argument(
+    "--offset",
+    type=int,
+    default=0,
+)
+parser.add_argument(
+    "--default",
+    type=str,
+    default="",
+)
+parser.add_argument(
+    "--delim",
+    type=str,
+    default=",",
+)
+parser.add_argument(
+    "--description",
+    type=str,
+    default="",
+)
+parser.add_argument(
+    "--destination_object",
+    type=str,
+    default="",
+)
+parser.add_argument(
+    "--object_name",
+    type=str,
+    default=os.getenv("abcli_object_name"),
+)
+parser.add_argument(
+    "--filter_string",
+    type=str,
+    help="tags.`keyword` = `value` - https://www.mlflow.org/docs/latest/search-runs.html",
+)
+parser.add_argument(
+    "--input",
+    type=str,
+    default="name",
+    help="id|name",
+)
+parser.add_argument(
+    "--item_name_plural",
+    type=str,
+    default="object(s)",
+)
+parser.add_argument(
+    "--model_name",
+    type=str,
+    default="",
+)
+parser.add_argument(
+    "--output",
+    type=str,
+    default="name",
+    help="id|name",
+)
+parser.add_argument(
+    "--regex",
+    type=str,
+    default="",
+)
+parser.add_argument(
+    "--log",
+    type=int,
+    default=1,
+    help="0|1",
+)
+parser.add_argument(
+    "--explicit_query",
+    type=int,
+    default=0,
+    help="0|1",
+)
+parser.add_argument(
+    "--start_end",
+    type=str,
+    default="",
+    help="start|end",
+)
+parser.add_argument(
+    "--stage_name",
+    type=str,
+    default="",
+    help="",
+)
+parser.add_argument(
+    "--tag",
+    type=str,
+    default="",
+)
+parser.add_argument(
+    "--source_objects",
+    type=str,
+    default="",
+)
+parser.add_argument(
+    "--tags",
+    type=str,
+    default="",
+    help="+this,that=which,what=12",
+)
+parser.add_argument(
+    "--value",
+    type=str,
+    default="",
+)
+parser.add_argument(
+    "--version",
+    type=str,
+    default="",
+    help="",
+)
+args = parser.parse_args()
+
+delim = " " if args.delim == "space" else args.delim
+
+success = False
+if args.task == "clone_tags":
+    success, tags = get_tags(args.source_objects)
+    if success:
+        success = set_tags(args.destination_object, tags)
+elif args.task == "create_filter_string":
+    success = True
+    print(create_filter_string(args.tags))
+elif args.task == "rm":
+    success = reduce(
+        lambda x, y: x and y,
+        [
+            rm(
+                object_name,
+                is_id=args.input == "id",
+            )
+            for object_name in args.object_name.split(",")
+            if object_name
+        ],
+        True,
+    )
+elif args.task == "get_tags":
+    success, tags = get_tags(args.object_name)
+    print(tags if not args.tag else tags.get(args.tag, args.default))
+elif args.task == "get_id":
+    success, id = get_id(args.object_name)
+    print(id)
+elif args.task == "get_run_id":
+    success, list_of_id = get_run_id(
+        args.object_name,
+        args.count,
+        args.offset,
+    )
+
+    print(delim.join(list_of_id))
+elif args.task == "list_registered_models":
+    success, list_of_models = list_registered_models()
+    if args.log:
+        logger.info(
+            "{:,} model(s): {}".format(len(list_of_models), delim.join(list_of_models))
+        )
+    else:
+        print(delim.join(list_of_models))
+elif args.task == "log_artifacts":
+    success = log_artifacts(
+        args.object_name,
+        args.model_name,
+    )
+elif args.task == "log_run":
+    success = log_run(args.object_name)
+elif args.task == "search":
+    success = True
+
+    filter_string = (
+        create_filter_string(args.tags)
+        if args.explicit_query == 0
+        else "" if args.filter_string == "-" else args.filter_string
+    )
+
+    list_of_objects = search(filter_string)
+
+    list_of_objects = list_of_objects[args.offset :]
+
+    if args.count != -1:
+        list_of_objects = list_of_objects[: args.count]
+
+    if args.log:
+        logger.info(
+            "{:,} {}.".format(
+                len(list_of_objects),
+                args.item_name_plural,
+            ),
+        )
+        for index, object_name in enumerate(list_of_objects):
+            logger.info(f"#{index+1: 4d} - {object_name}")
+    else:
+        print(delim.join(list_of_objects))
+elif args.task == "set_tags":
+    success = set_tags(
+        args.object_name,
+        args.tags,
+    )
+elif args.task == "transition":
+    success = transition(
+        args.model_name,
+        args.version,
+        args.stage_name,
+        description=args.description,
+    )
+elif args.task == "start_end_run":
+    if args.start_end == "start":
+        success = start_run(args.object_name)
+    elif args.task == "end":
+        success = end_run(args.object_name)
+    else:
+        success = False
+        logger.info(f"expected start|end,received {args.start_end}.")
+elif args.task == "test":
+    success = test()
+else:
+    success = None
+
+sys_exit(logger, NAME, args.task, success)

bluer_objects/mlflow/cache.py

@@ -0,0 +1,13 @@
+from typing import Tuple
+
+from bluer_objects.mlflow.tags import get_tags, set_tags
+
+
+def read(keyword: str) -> Tuple[bool, str]:
+    success, tags = get_tags(keyword)
+
+    return success, tags.get("referent", "")
+
+
+def write(keyword: str, value: str) -> bool:
+    return set_tags(keyword, {"referent": value})