bluer-objects 6.104.1__py3-none-any.whl → 6.464.1__py3-none-any.whl

bluer_objects/mlflow/serverless/write.py

@@ -0,0 +1,42 @@
+from typing import Dict, Union
+
+from blueness import module
+from bluer_options.options import Options
+
+from bluer_objects import NAME
+from bluer_objects.mlflow.serverless import api
+from bluer_objects.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+
+def set_tags(
+    object_name: str,
+    tags: Union[str, Dict[str, str]],
+    log: bool = True,
+    verbose: bool = False,
+    icon="#️⃣ ",
+) -> bool:
+    tags = Options(tags)
+
+    if not api.write(
+        object_name="_serverless_objects",
+        filename=f"{object_name}.yaml",
+        data=tags,
+        log=verbose,
+    ):
+        return False
+
+    for key, value in tags.items():
+        if not api.write(
+            object_name="_serverless_keys",
+            filename=f"{key}.yaml",
+            data={object_name: value},
+            log=verbose,
+        ):
+            return False
+
+        if log:
+            logger.info("{} {}.{}={}".format(icon, object_name, key, value))
+
+    return True

bluer_objects/mlflow/tags.py

@@ -7,13 +7,17 @@ from bluer_options.options import Options
 from bluer_options.logger import crash_report
 
 from bluer_objects import NAME
+from bluer_objects import env
+from bluer_objects.mlflow import serverless
 from bluer_objects.mlflow.objects import to_experiment_name, to_object_name
 from bluer_objects.logger import logger
 
 NAME = module.name(__file__, NAME)
 
 
-def create_filter_string(tags: str) -> str:
+def create_server_style_filter_string(
+    tags: Union[str, dict],
+) -> str:
     tags_options = Options(tags)
 
     # https://www.mlflow.org/docs/latest/search-experiments.html
@@ -25,7 +29,14 @@ def create_filter_string(tags: str) -> str:
 def get_tags(
     object_name: str,
     exclude_system_tags: bool = True,
+    verbose: bool = False,
 ) -> Tuple[bool, Dict[str, str]]:
+    if env.MLFLOW_IS_SERVERLESS:
+        return serverless.get_tags(
+            object_name,
+            verbose=verbose,
+        )
+
     experiment_name = to_experiment_name(object_name)
 
     try:
@@ -48,16 +59,45 @@ def get_tags(
 
 
 # https://www.mlflow.org/docs/latest/search-experiments.html
-def search(filter_string: str) -> List[str]:
-    client = MlflowClient()
-
-    return [
-        to_object_name(experiment.name)
-        for experiment in client.search_experiments(
+def search(
+    filter_string: Union[str, dict],
+    server_style: bool = False,
+    verbose: bool = False,
+) -> Tuple[bool, List[str]]:
+    if server_style and env.MLFLOW_IS_SERVERLESS:
+        logger.error("server_style search is not supported when mlflow is serverless.")
+        return False, []
+
+    if filter_string == "-":
+        filter_string = ""
+
+    if env.MLFLOW_IS_SERVERLESS:
+        return serverless.search(
             filter_string=filter_string,
-            view_type=ViewType.ALL,
+            log=verbose,
         )
-    ]
+
+    if not server_style:
+        filter_string = create_server_style_filter_string(filter_string)
+
+    client = MlflowClient()
+
+    success = False
+    output = []
+
+    try:
+        output = [
+            to_object_name(experiment.name)
+            for experiment in client.search_experiments(
+                filter_string=filter_string,
+                view_type=ViewType.ALL,
+            )
+        ]
+        success = True
+    except Exception as e:
+        logger.error(e)
+
+    return success, output
 
 
 def set_tags(
@@ -65,7 +105,17 @@ def set_tags(
     tags: Union[str, Dict[str, str]],
     log: bool = True,
     icon="#️⃣ ",
+    verbose: bool = False,
 ) -> bool:
+    if env.MLFLOW_IS_SERVERLESS:
+        return serverless.set_tags(
+            object_name,
+            tags=tags,
+            log=log,
+            verbose=verbose,
+            icon=icon,
+        )
+
     experiment_name = to_experiment_name(object_name)
 
     try:

bluer_objects/objects.py

@@ -69,6 +69,7 @@ def signature(info=None, object_name="."):
 def unique_object(
     prefix: str = "",
     include_time: bool = True,
+    log: bool = True,
 ):
     object_name = string.pretty_date(
         as_filename=True,
@@ -80,6 +81,7 @@ def unique_object(
 
     path.create(object_path(object_name))
 
-    logger.info(f"📂 {object_name}")
+    if log:
+        logger.info(f"📂 {object_name}")
 
     return object_name

bluer_objects/pdf/__init__.py

@@ -0,0 +1 @@
+

bluer_objects/pdf/__main__.py

@@ -0,0 +1,78 @@
+import argparse
+
+from blueness import module
+from blueness.argparse.generic import sys_exit
+
+from bluer_objects import NAME
+from bluer_objects.pdf.convert.batch import batch as batch_convert
+from bluer_objects.pdf.convert.convert import convert
+from bluer_objects.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+parser = argparse.ArgumentParser(NAME)
+parser.add_argument(
+    "task",
+    type=str,
+    help="convert",
+)
+parser.add_argument(
+    "--path_prefix",
+    type=str,
+)
+parser.add_argument(
+    "--object_name",
+    type=str,
+)
+parser.add_argument(
+    "--suffixes",
+    type=str,
+)
+parser.add_argument(
+    "--combine",
+    type=int,
+    default=0,
+    help="0 | 1",
+)
+
+parser.add_argument(
+    "--count",
+    type=int,
+    default=-1,
+    help="-1: all",
+)
+parser.add_argument(
+    "--list_missing",
+    type=int,
+    default=1,
+    help="0 | 1",
+)
+parser.add_argument(
+    "--use_metadata",
+    type=int,
+    default=0,
+    help="0 | 1",
+)
+args = parser.parse_args()
+
+success = False
+if args.task == "convert":
+    if args.use_metadata == 1:
+        success = batch_convert(
+            object_name=args.object_name,
+            combine=args.combine == 1,
+            count=args.count,
+            list_missing=args.list_missing == 1,
+        )
+    else:
+        success = convert(
+            path_prefix=args.path_prefix,
+            list_of_suffixes=args.suffixes.split(","),
+            object_name=args.object_name,
+            combine=args.combine == 1,
+            count=args.count,
+        )
+else:
+    success = None
+
+sys_exit(logger, NAME, args.task, success)

bluer_objects/pdf/convert/batch.py

@@ -0,0 +1,54 @@
+from typing import List
+
+from blueness import module
+
+from bluer_objects import NAME
+from bluer_objects.metadata import get_from_object
+from bluer_objects.pdf.convert.convert import convert
+from bluer_objects.pdf.convert.missing import list_missing_docs
+from bluer_objects.env import abcli_path_git
+from bluer_objects.logger import logger
+
+
+NAME = module.name(__file__, NAME)
+
+
+def batch(
+    object_name: str,
+    combine: bool,
+    count: int = -1,
+    list_missing: bool = True,
+) -> bool:
+    logger.info(
+        "{}.batch: {}{}{}{}".format(
+            NAME,
+            object_name,
+            " + combine" if combine else "",
+            "" if count == -1 else f" {count} files",
+            " + list missing" if list_missing else "",
+        )
+    )
+
+    list_of_suffixes: List[str] = get_from_object(
+        object_name,
+        "pdf",
+        [],
+    )
+
+    if not convert(
+        path_prefix=abcli_path_git,
+        list_of_suffixes=list_of_suffixes,
+        object_name=object_name,
+        combine=combine,
+        count=count,
+        incremental=False,
+    ):
+        return False
+
+    if list_missing:
+        list_missing_docs(
+            object_name=object_name,
+            list_of_suffixes=list_of_suffixes,
+        )
+
+    return True

bluer_objects/pdf/convert/combination.py

@@ -0,0 +1,32 @@
+from typing import List
+from PyPDF2 import PdfMerger
+
+from bluer_options.logger import crash_report
+
+from bluer_objects import objects
+from bluer_objects.logger import logger
+
+
+def combine_pdfs(
+    list_of_pdfs: List[str],
+    object_name: str,
+) -> bool:
+    logger.info(f"combining {len(list_of_pdfs)} pdf(s)...")
+    combined_filename = objects.path_of(
+        filename="release.pdf",
+        object_name=object_name,
+    )
+
+    try:
+        merger = PdfMerger()
+        for filename in list_of_pdfs:
+            merger.append(filename)
+
+        merger.write(combined_filename)
+        merger.close()
+    except Exception as e:
+        crash_report(e)
+        return False
+
+    logger.info(f"-> {combined_filename}")
+    return True

bluer_objects/pdf/convert/convert.py

@@ -0,0 +1,110 @@
+from tqdm import tqdm
+from typing import List
+import os
+from PIL import Image
+
+from blueness import module
+
+from bluer_objects import NAME
+from bluer_objects import file, path
+from bluer_objects.metadata import get_from_object, post_to_object
+from bluer_objects.pdf.convert.combination import combine_pdfs
+from bluer_objects.pdf.convert.image import convert_image
+from bluer_objects.pdf.convert.md import convert_md
+from bluer_objects.pdf.convert.pdf import convert_pdf
+from bluer_objects.logger import logger
+
+
+NAME = module.name(__file__, NAME)
+
+
+def convert(
+    path_prefix: str,
+    list_of_suffixes: List[str],
+    object_name: str,
+    combine: bool,
+    count: int = -1,
+    incremental: bool = True,
+) -> bool:
+    logger.info(f"path_prefix: {path_prefix}")
+
+    list_of_pdfs: List[str] = (
+        get_from_object(
+            object_name,
+            "list_of_pdfs",
+            [],
+        )
+        if incremental
+        else []
+    )
+    if incremental:
+        logger.info(f"found {len(list_of_pdfs)} pdf(s)...")
+
+    list_of_pdfs_len_target = -1 if count == -1 else len(list_of_pdfs) + count
+    for suffix in tqdm(list_of_suffixes):
+        if (
+            list_of_pdfs_len_target != -1
+            and len(list_of_pdfs) >= list_of_pdfs_len_target
+        ):
+            logger.info(f"max count {count}, stopping.")
+            break
+
+        logger.info(
+            "{}.convert {} -> {}".format(
+                NAME,
+                suffix,
+                object_name,
+            )
+        )
+
+        source_filename = os.path.join(path_prefix, suffix)
+        if path.exists(source_filename):
+            source_filename = os.path.join(source_filename, "README.md")
+            suffix = os.path.join(suffix, "README.md")
+
+        if source_filename.endswith(".md"):
+            if not convert_md(
+                source_filename,
+                suffix,
+                object_name,
+                list_of_pdfs,
+            ):
+                return False
+        elif file.extension(source_filename) == "pdf":
+            if not convert_pdf(
+                source_filename,
+                suffix,
+                object_name,
+                list_of_pdfs,
+            ):
+                return False
+        elif file.extension(source_filename) in [
+            extension.split(".", 1)[1] for extension in Image.registered_extensions()
+        ]:
+            if not convert_image(
+                source_filename,
+                suffix,
+                object_name,
+                list_of_pdfs,
+            ):
+                return False
+        else:
+            logger.error(f"{source_filename}: cannot convert to pdf.")
+            return False
+
+    if incremental:
+        logger.info(f"{len(list_of_pdfs)} pdf(s) so far ...")
+        if not post_to_object(
+            object_name,
+            "list_of_pdfs",
+            list_of_pdfs,
+        ):
+            return False
+
+    if combine:
+        return combine_pdfs(
+            list_of_pdfs,
+            object_name,
+        )
+
+    return True

bluer_objects/pdf/convert/image.py

@@ -0,0 +1,53 @@
+from PIL import Image
+
+from bluer_options.logger import crash_report
+
+from bluer_objects import file, objects, path
+from bluer_objects.env import abcli_path_git
+from bluer_objects.logger import logger
+
+
+def convert_image(
+    source_filename: str,
+    suffix: str,
+    object_name: str,
+    list_of_pdfs: list[str],
+) -> bool:
+    logger.info("🌠 image found!")
+    filename_pdf = file.add_extension(
+        objects.path_of(
+            filename="docs/{}".format(
+                (
+                    suffix.split(abcli_path_git, 1)[1]
+                    if abcli_path_git in suffix
+                    else suffix
+                ),
+            ),
+            object_name=object_name,
+        ),
+        "pdf",
+    )
+
+    if filename_pdf not in list_of_pdfs:
+        list_of_pdfs.append(filename_pdf)
+
+    if file.exists(filename_pdf):
+        logger.info(f"✅ {filename_pdf}")
+        return True
+
+    if not path.create(
+        file.path(filename_pdf),
+        log=True,
+    ):
+        return False
+
+    try:
+        image = Image.open(source_filename)
+        image = image.convert("RGB")
+        image.save(filename_pdf)
+    except Exception as e:
+        crash_report(e)
+        return False
+
+    logger.info(f"-> {filename_pdf}")
+    return True

bluer_objects/pdf/convert/md.py

@@ -0,0 +1,97 @@
+import pypandoc
+import subprocess
+import os
+
+from bluer_options.logger import crash_report
+
+from bluer_objects import file, objects, path
+from bluer_objects.env import abcli_path_git
+from bluer_objects.logger import logger
+
+css = """
+    <style>
+        body { font-family: sans-serif; margin: 2cm; }
+        img { max-width: 100%; height: auto; }
+        table { width: 100%; border-collapse: collapse; word-break: break-word; }
+        th, td { border: 1px solid #ccc; padding: 4px; vertical-align: top; }
+        code, pre { white-space: pre-wrap; }
+    </style>
+    """
+
+
+def convert_md(
+    source_filename: str,
+    suffix: str,
+    object_name: str,
+    list_of_pdfs: list[str],
+) -> bool:
+    filename_html = file.add_extension(
+        objects.path_of(
+            filename="docs/{}".format(
+                (
+                    suffix.split(abcli_path_git, 1)[1]
+                    if abcli_path_git in suffix
+                    else suffix
+                ),
+            ),
+            object_name=object_name,
+        ),
+        "html",
+    )
+    filename_pdf = file.add_extension(
+        filename_html,
+        "pdf",
+    )
+
+    if filename_pdf not in list_of_pdfs:
+        list_of_pdfs.append(filename_pdf)
+
+    if file.exists(filename_pdf):
+        logger.info(f"✅ {filename_pdf}")
+        return True
+
+    logger.info(f"{source_filename} -> {filename_pdf}")
+
+    try:
+        with open(source_filename, "r", encoding="utf-8") as f:
+            markdown_text = f.read()
+
+        html_text = pypandoc.convert_text(
+            markdown_text,
+            "html",
+            format="md",
+        )
+
+        html_text = (
+            f"<!DOCTYPE html><html><head>{css}</head><body>{html_text}</body></html>"
+        )
+
+        if not path.create(
+            file.path(filename_html),
+            log=True,
+        ):
+            return (False,)
+
+        with open(
+            filename_html,
+            "w",
+            encoding="utf-8",
+        ) as f:
+            f.write(html_text)
+
+        subprocess.run(
+            [
+                "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
+                "--headless",
+                "--disable-gpu",
+                "--no-margins",
+                f"--print-to-pdf={filename_pdf}",
+                os.path.abspath(filename_html),
+            ],
+            check=True,
+        )
+    except Exception as e:
+        crash_report(e)
+        return False
+
+    return True

bluer_objects/pdf/convert/missing.py

@@ -0,0 +1,96 @@
+from typing import List
+import os
+from functools import reduce
+import re
+
+from blueness import module
+from bluer_options.logger.config import log_list
+
+from bluer_objects import NAME
+from bluer_objects import file
+from bluer_objects.metadata import get_from_object
+from bluer_objects.pdf.convert.convert import convert
+from bluer_objects.env import abcli_path_git
+from bluer_objects.logger import logger
+
+
+def list_missing_docs(
+    object_name: str,
+    list_of_suffixes: List[str],
+):
+    logger.info(f"{NAME}.list_missing_docs: {object_name}")
+
+    list_of_suffixes_fullpath = [
+        os.path.join(abcli_path_git, suffix) for suffix in list_of_suffixes
+    ]
+
+    list_of_prefixes_to_ignore: List[str] = [
+        os.path.join(abcli_path_git, prefix)
+        for prefix in get_from_object(
+            object_name,
+            "ignore",
+            [],
+        )
+    ]
+    log_list(
+        logger,
+        "ignoring",
+        list_of_prefixes_to_ignore,
+        "prefix(s)",
+        max_count=-1,
+    )
+
+    list_of_prefixes: List[str] = sorted(
+        list({suffix.split(os.sep)[0] for suffix in list_of_suffixes})
+    )
+    log_list(
+        logger,
+        "found",
+        list_of_prefixes,
+        "prefix(s)",
+        max_count=-1,
+    )
+
+    list_of_missing_docs: List[str] = [
+        filename
+        for filename in reduce(
+            lambda x, y: x + y,
+            [
+                file.list_of(
+                    os.path.join(
+                        abcli_path_git,
+                        prefix,
+                        "*.md",
+                    ),
+                    recursive=True,
+                )
+                for prefix in list_of_prefixes
+            ],
+            [],
+        )
+        if not any(
+            thing in filename
+            for thing in [
+                "template",
+                "pytest_cache",
+            ]
+        )
+        and filename not in list_of_suffixes_fullpath
+        and not any(
+            filename.startswith(prefix) for prefix in list_of_prefixes_to_ignore
+        )
+        and re.search(r"(?:^|-)v(\d+)\.md$", file.name_and_extension(filename)) is None
+        and file.name_and_extension(filename) != "README.md"
+    ]
+    if list_of_missing_docs:
+        logger.warning("missing docs.")
+        log_list(
+            logger,
+            "found",
+            list_of_missing_docs,
+            "missing doc(s)",
+            max_count=-1,
+            max_length=-1,
+        )
+    else:
+        logger.info("✅ no missing docs.")