bluer-objects 6.3.1__py3-none-any.whl

bluer_objects/mlflow/logging.py

@@ -0,0 +1,81 @@
+from typing import Dict
+import os
+import glob
+import mlflow
+
+from blueness import module
+from blue_options.logger import crash_report
+
+from bluer_objects import file, objects, NAME
+from bluer_objects.mlflow.runs import start_run, end_run
+from bluer_objects.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+
+def log_artifacts(
+    object_name: str,
+    model_name: str = "",
+) -> bool:
+    if not start_run(object_name):
+        return False
+
+    object_path = objects.object_path(object_name, create=True)
+
+    try:
+        mlflow.log_artifacts(object_path)
+
+        logger.info("⬆️  {}".format(object_name))
+
+        # https://mlflow.org/docs/latest/python_api/mlflow.html#mlflow.register_model
+        # https://stackoverflow.com/a/71447758/17619982
+        if model_name:
+            mv = mlflow.register_model(
+                "runs:/{}".format(mlflow.active_run().info.run_id),
+                model_name,
+                await_registration_for=0,
+            )
+
+            logger.info("*️⃣  {} -> {}.{}".format(object_name, mv.name, mv.version))
+
+    except:
+        crash_report(f"{NAME}.log_artifacts({object_name})")
+        return False
+
+    return end_run(object_name)
+
+
+def log_run(object_name: str) -> bool:
+    if not start_run(object_name):
+        return False
+
+    object_path = objects.object_path(object_name, create=True)
+
+    counts: Dict[str, int] = {}
+    skipped_count = 0
+    for extension in "dot,gif,jpeg,jpg,json,png,sh,xml,yaml".split(","):
+        for filename in glob.glob(
+            os.path.join(object_path, f"*.{extension}"),
+        ):
+            filename_name = file.name(filename)
+
+            counts[len(filename_name)] = counts.get(len(filename_name), 0) + 1
+
+            if any(
+                [
+                    file.size(filename) > 10 * 1024 * 1024,
+                    filename_name.startswith("thumbnail"),
+                    counts[len(filename_name)] > 20,
+                ]
+            ):
+                logger.info(f"skipping {filename}")
+                skipped_count += 1
+                continue
+
+            mlflow.log_artifact(filename)
+            logger.info(f"⬆️  {filename}")
+
+    if skipped_count:
+        logger.info(f"skipped {skipped_count:,} file(s).")
+
+    return end_run(object_name)

bluer_objects/mlflow/models.py

@@ -0,0 +1,57 @@
+from typing import Tuple, List
+from mlflow.tracking import MlflowClient
+
+from blueness import module
+from blue_options.logger import crash_report
+
+from bluer_objects import NAME
+from bluer_objects.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+
+def list_registered_models() -> Tuple[
+    bool,
+    List[str],
+]:
+    try:
+        client = MlflowClient()
+        return True, [dict(rm)["name"] for rm in client.search_registered_models()]
+
+    except:
+        crash_report(f"{NAME}.list_registered_models()")
+        return False, []
+
+
+def transition(
+    model_name: str,
+    version: str,
+    stage_name: str,
+    description: str,
+) -> bool:
+    logger.info(
+        '{}.transition: {}(#{}) ->  {} - "{}")'.format(
+            NAME,
+            model_name,
+            version,
+            stage_name,
+            description,
+        )
+    )
+
+    try:
+        client = MlflowClient()
+        client.transition_model_version_stage(
+            name=model_name, version=version, stage=stage_name
+        )
+
+        if description:
+            client.update_model_version(
+                name=model_name, version=version, description=description
+            )
+
+    except:
+        crash_report(f"{NAME}.transition({model_name})")
+        return False
+
+    return True

bluer_objects/mlflow/objects.py

@@ -0,0 +1,76 @@
+from typing import Tuple
+import mlflow
+from mlflow.tracking import MlflowClient
+
+from blueness import module
+from blue_options.logger import crash_report
+
+from bluer_objects import NAME
+from bluer_objects.env import ABCLI_MLFLOW_EXPERIMENT_PREFIX
+from bluer_objects.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+
+def get_id(
+    object_name: str,
+    create: bool = False,
+) -> Tuple[bool, str]:
+    experiment_name = to_experiment_name(object_name)
+
+    try:
+        experiment = mlflow.get_experiment_by_name(experiment_name)
+        if experiment is None:
+            if create:
+                MlflowClient().create_experiment(name=experiment_name)
+                experiment = mlflow.get_experiment_by_name(experiment_name)
+            else:
+                return True, ""
+
+        return True, dict(experiment)["experiment_id"]
+    except:
+        crash_report(f"{NAME}.get_id({object_name})")
+
+        return False, ""
+
+
+def rm(
+    object_name: str,
+    is_id: bool = False,
+) -> bool:
+    if is_id:
+        experiment_id = object_name
+    else:
+        success, experiment_id = get_id(object_name)
+        if not success:
+            return success
+
+    try:
+        client = MlflowClient()
+
+        # get list of run_ids
+
+        # delete all runs
+
+        client.delete_experiment(experiment_id)
+    except:
+        crash_report("mlflow.rm({})".format(object_name))
+        return False
+
+    logger.info(
+        "🚮 {}".format(
+            "#{}".format(experiment_id)
+            if is_id
+            else "{} (#{})".format(object_name, experiment_id)
+        )
+    )
+
+    return True
+
+
+def to_experiment_name(object_name: str) -> str:
+    return f"{ABCLI_MLFLOW_EXPERIMENT_PREFIX}{object_name}"
+
+
+def to_object_name(experiment_name: str) -> str:
+    return experiment_name.split(ABCLI_MLFLOW_EXPERIMENT_PREFIX)[-1]

bluer_objects/mlflow/runs.py

@@ -0,0 +1,100 @@
+from typing import Tuple, List
+import mlflow
+import random
+from mlflow.tracking import MlflowClient
+from mlflow.entities import ViewType
+
+from blueness import module
+from blue_options import string
+from blue_options.logger import crash_report
+
+from bluer_objects import NAME
+from bluer_objects.mlflow.objects import get_id
+from bluer_objects.mlflow.tags import get_tags
+from bluer_objects.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+
+def end_run(
+    object_name: str,
+) -> bool:
+    try:
+        mlflow.end_run()
+        logger.info("⏹️  {}".format(object_name))
+    except:
+        crash_report(f"{NAME}.end_run({object_name})")
+        return False
+
+    return True
+
+
+def get_run_id(
+    object_name: str,
+    count: int = -1,
+    offset: int = 0,
+    create: bool = False,
+    is_id: bool = False,
+) -> Tuple[bool, List[str]]:
+    if is_id:
+        experiment_id = object_name
+    else:
+        success, experiment_id = get_id(object_name, create=create)
+        if not success:
+            return False, []
+
+    try:
+        client = MlflowClient()
+
+        list_of_runs = client.search_runs(
+            experiment_ids=[experiment_id],
+            run_view_type=ViewType.ACTIVE_ONLY,
+            max_results=count + offset,
+        )
+
+        return True, [run._info.run_id for run in list_of_runs][offset:]
+
+    except:
+        crash_report(f"{NAME}.get_run_id({object_name})")
+        return False, []
+
+
+def start_run(
+    object_name: str,
+) -> bool:
+    success, experiment_id = get_id(object_name, create=True)
+    if not success:
+        return False
+
+    max_count = 25000
+    success, list_of_runs = get_run_id(
+        experiment_id,
+        count=max_count,
+        is_id=True,
+    )
+    if not success:
+        return False
+
+    run_counter = len(list_of_runs) + 1
+    if run_counter > max_count:
+        logger.warning(f"{object_name}: more than {max_count} runs!")
+        run_counter = max_count + random.randint(1, max_count)
+
+    run_name = f"{object_name}-{run_counter:05d}"
+
+    success, tags = get_tags(object_name)
+    if not success:
+        return False
+
+    try:
+        mlflow.start_run(
+            experiment_id=experiment_id,
+            tags=tags,
+            run_name=run_name,
+        )
+        logger.info(f"⏺️  {object_name} | {run_counter:05d}")
+    except:
+        crash_report(f"{NAME}.start_run({object_name})")
+        return False
+
+    return True

bluer_objects/mlflow/tags.py

@@ -0,0 +1,90 @@
+from typing import Tuple, Dict, List, Union
+from mlflow.tracking import MlflowClient
+from mlflow.entities import ViewType
+
+from blueness import module
+from blue_options.options import Options
+from blue_options.logger import crash_report
+
+from bluer_objects import NAME
+from bluer_objects.mlflow.objects import to_experiment_name, to_object_name
+from bluer_objects.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+
+def create_filter_string(tags: str) -> str:
+    tags_options = Options(tags)
+
+    # https://www.mlflow.org/docs/latest/search-experiments.html
+    return " and ".join(
+        [f'tags."{keyword}" = "{value}"' for keyword, value in tags_options.items()]
+    )
+
+
+def get_tags(
+    object_name: str,
+    exclude_system_tags: bool = True,
+) -> Tuple[bool, Dict[str, str]]:
+    experiment_name = to_experiment_name(object_name)
+
+    try:
+        client = MlflowClient()
+        experiment = client.get_experiment_by_name(experiment_name)
+
+        if experiment is None:
+            return True, {}
+
+        tags = {
+            keyword: value
+            for keyword, value in experiment.tags.items()
+            if not keyword.startswith("mlflow.") or not exclude_system_tags
+        }
+
+        return True, tags
+    except:
+        crash_report(f"{NAME}.get_tags({object_name})")
+        return False, {}
+
+
+# https://www.mlflow.org/docs/latest/search-experiments.html
+def search(filter_string: str) -> List[str]:
+    client = MlflowClient()
+
+    return [
+        to_object_name(experiment.name)
+        for experiment in client.search_experiments(
+            filter_string=filter_string,
+            view_type=ViewType.ALL,
+        )
+    ]
+
+
+def set_tags(
+    object_name: str,
+    tags: Union[str, Dict[str, str]],
+    log: bool = True,
+    icon="#️⃣ ",
+) -> bool:
+    experiment_name = to_experiment_name(object_name)
+
+    try:
+        tags = Options(tags)
+
+        client = MlflowClient()
+
+        experiment = client.get_experiment_by_name(experiment_name)
+        if experiment is None:
+            client.create_experiment(name=experiment_name)
+            experiment = client.get_experiment_by_name(experiment_name)
+
+        for key, value in tags.items():
+            client.set_experiment_tag(experiment.experiment_id, key, value)
+            if log:
+                logger.info("{} {}.{}={}".format(icon, object_name, key, value))
+
+    except:
+        crash_report(f"{NAME}.set_tags({object_name})")
+        return False
+
+    return True

bluer_objects/mlflow/testing.py

@@ -0,0 +1,39 @@
+import mlflow
+
+from blueness import module
+from blue_options import string
+from blue_options.logger import crash_report
+
+from bluer_objects import NAME, VERSION
+from bluer_objects.mlflow.objects import get_id
+from bluer_objects.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+
+def test() -> bool:
+    object_name = string.pretty_date(
+        as_filename=True,
+        unique=True,
+    )
+
+    success, experiment_id = get_id(
+        object_name,
+        create=True,
+    )
+    if not success:
+        return success
+
+    try:
+        mlflow.start_run(
+            experiment_id=experiment_id,
+            tags={"purpose": "testing"},
+        )
+        mlflow.end_run()
+
+        logger.info(f"✅ {NAME}-{VERSION}-{mlflow.version.VERSION}")
+    except:
+        crash_report(f"{NAME}.test()")
+        return False
+
+    return True

bluer_objects/mysql/cache/__init__.py

@@ -0,0 +1,8 @@
+from bluer_objects.mysql.cache.functions import (
+    create,
+    clone,
+    read,
+    write,
+    search,
+    search_value,
+)

bluer_objects/mysql/cache/__main__.py

@@ -0,0 +1,91 @@
+import argparse
+
+from blueness import module
+from blueness.argparse.generic import sys_exit
+
+from bluer_objects import NAME
+from bluer_objects.mysql.cache.functions import create, clone, read, write, search
+from bluer_objects.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+
+parser = argparse.ArgumentParser(NAME)
+parser.add_argument(
+    "task",
+    type=str,
+    default="read",
+    help="clone,create,read,search,write",
+)
+parser.add_argument(
+    "--all",
+    default=0,
+    type=int,
+)
+parser.add_argument(
+    "--dataframe",
+    default=0,
+    type=int,
+)
+parser.add_argument(
+    "--destination",
+    type=str,
+    default="",
+)
+parser.add_argument(
+    "--keyword",
+    type=str,
+)
+parser.add_argument(
+    "--like",
+    default=0,
+    type=int,
+)
+parser.add_argument(
+    "--source",
+    type=str,
+    default="",
+)
+parser.add_argument(
+    "--unique",
+    default=1,
+    type=int,
+    help="cache.read('unique')",
+)
+parser.add_argument(
+    "--value",
+    type=str,
+    default="unknown",
+)
+args = parser.parse_args()
+
+success = False
+if args.task == "clone":
+    success = clone(args.source, args.destination)
+elif args.task == "create":
+    success = create()
+elif args.task == "read":
+    output = read(
+        args.keyword,
+        all=args.all,
+        like=args.like,
+        dataframe=args.dataframe,
+        unique=args.unique,
+    )
+
+    print(
+        ("None" if output.empty else output.drop(columns=["log_id"]))
+        if args.dataframe
+        else output
+    )
+    success = True
+elif args.task == "search":
+    for keyword, value in search(args.keyword).items():
+        print(f"{keyword}: {value}")
+    success = True
+elif args.task == "write":
+    success = write(args.keyword, args.value)
+else:
+    success = None
+
+sys_exit(logger, NAME, args.task, success)