bluer-objects 6.377.1__py3-none-any.whl → 6.464.1__py3-none-any.whl

bluer_objects/mlflow/objects.py

@@ -6,6 +6,7 @@ from blueness import module
 from bluer_options.logger import crash_report
 
 from bluer_objects import NAME
+from bluer_objects import env
 from bluer_objects.env import ABCLI_MLFLOW_EXPERIMENT_PREFIX
 from bluer_objects.logger import logger
 
@@ -16,6 +17,9 @@ def get_id(
     object_name: str,
     create: bool = False,
 ) -> Tuple[bool, str]:
+    if env.MLFLOW_IS_SERVERLESS:
+        return True, "serverless-id"
+
     experiment_name = to_experiment_name(object_name)
 
     try:
@@ -38,6 +42,9 @@ def rm(
     object_name: str,
     is_id: bool = False,
 ) -> bool:
+    if env.MLFLOW_IS_SERVERLESS:
+        return True
+
     if is_id:
         experiment_id = object_name
     else:

bluer_objects/mlflow/runs.py

@@ -5,10 +5,10 @@ from mlflow.tracking import MlflowClient
 from mlflow.entities import ViewType
 
 from blueness import module
-from bluer_options import string
 from bluer_options.logger import crash_report
 
 from bluer_objects import NAME
+from bluer_objects import env
 from bluer_objects.mlflow.objects import get_id
 from bluer_objects.mlflow.tags import get_tags
 from bluer_objects.logger import logger
@@ -19,6 +19,9 @@ NAME = module.name(__file__, NAME)
 def end_run(
     object_name: str,
 ) -> bool:
+    if env.MLFLOW_IS_SERVERLESS:
+        return True
+
     try:
         mlflow.end_run()
         logger.info("⏹️  {}".format(object_name))
@@ -36,6 +39,9 @@ def get_run_id(
     create: bool = False,
     is_id: bool = False,
 ) -> Tuple[bool, List[str]]:
+    if env.MLFLOW_IS_SERVERLESS:
+        return True, []
+
     if is_id:
         experiment_id = object_name
     else:
@@ -62,6 +68,9 @@ def get_run_id(
 def start_run(
     object_name: str,
 ) -> bool:
+    if env.MLFLOW_IS_SERVERLESS:
+        return True
+
     success, experiment_id = get_id(object_name, create=True)
     if not success:
         return False

bluer_objects/mlflow/serverless/__init__.py

@@ -0,0 +1,3 @@
+from bluer_objects.mlflow.serverless.read import get_tags
+from bluer_objects.mlflow.serverless.search import search
+from bluer_objects.mlflow.serverless.write import set_tags

bluer_objects/mlflow/serverless/api.py

@@ -0,0 +1,88 @@
+from typing import Dict, Tuple
+
+from blueness import module
+
+from bluer_objects import NAME
+from bluer_objects import file
+from bluer_objects import objects
+from bluer_objects import storage
+from bluer_objects.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+
+def write(
+    object_name: str,
+    filename: str,
+    data: Dict,
+    log: bool = False,
+) -> bool:
+    if log:
+        logger.info(f"{NAME}.update({object_name}/{filename})")
+
+    if not storage.download(
+        object_name=object_name,
+        filename=filename,
+        log=log,
+    ):
+        return False
+
+    full_filename = objects.path_of(
+        object_name=object_name,
+        filename=filename,
+    )
+
+    _, current_data = file.load_yaml(
+        full_filename,
+        ignore_error=True,
+        default={},
+    )
+    if not isinstance(current_data, dict):
+        logger.error(
+            "dict expected, {} received".format(
+                current_data.__class__.__name__,
+            )
+        )
+        return False
+
+    current_data.update(data)
+
+    if not file.save_yaml(
+        full_filename,
+        current_data,
+        log=log,
+    ):
+        return False
+
+    if not storage.upload(
+        object_name=object_name,
+        filename=filename,
+        log=log,
+    ):
+        return False
+
+    return True
+
+
+def read(
+    object_name: str,
+    filename: str,
+    verbose: bool = False,
+) -> Tuple[bool, Dict]:
+    if not storage.download(
+        object_name=object_name,
+        filename=filename,
+        log=verbose,
+    ):
+        return True, {}
+
+    _, data = file.load_yaml(
+        objects.path_of(
+            object_name=object_name,
+            filename=filename,
+        ),
+        ignore_error=True,
+        default={},
+    )
+
+    return True, data

bluer_objects/mlflow/serverless/read.py

@@ -0,0 +1,19 @@
+from typing import Tuple, Dict
+
+from blueness import module
+
+from bluer_objects import NAME
+from bluer_objects.mlflow.serverless import api
+
+NAME = module.name(__file__, NAME)
+
+
+def get_tags(
+    object_name: str,
+    verbose: bool = False,
+) -> Tuple[bool, Dict[str, str]]:
+    return api.read(
+        object_name="_serverless_objects",
+        filename=f"{object_name}.yaml",
+        verbose=verbose,
+    )

bluer_objects/mlflow/serverless/search.py

@@ -0,0 +1,35 @@
+from typing import List, Tuple, Union
+
+from bluer_options.options import Options
+
+from bluer_objects.mlflow.serverless import api
+from bluer_objects.logger import logger
+
+
+def search(
+    filter_string: Union[str, dict],
+    log: bool = False,
+) -> Tuple[bool, List[str]]:
+    query = Options(filter_string)
+
+    output: Union[List[str], None] = None
+    for key, value in query.items():
+        success, dict_of_objects = api.read(
+            object_name="_serverless_keys",
+            filename=f"{key}.yaml",
+            verbose=log,
+        )
+        if not success:
+            return False, []
+
+        output = [
+            object_name
+            for object_name in (dict_of_objects if output is None else output)
+            if dict_of_objects.get(
+                object_name,
+                False if isinstance(value, bool) else None,
+            )
+            == value
+        ]
+
+    return True, [] if output is None else output

bluer_objects/mlflow/serverless/write.py

@@ -0,0 +1,42 @@
+from typing import Dict, Union
+
+from blueness import module
+from bluer_options.options import Options
+
+from bluer_objects import NAME
+from bluer_objects.mlflow.serverless import api
+from bluer_objects.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+
+def set_tags(
+    object_name: str,
+    tags: Union[str, Dict[str, str]],
+    log: bool = True,
+    verbose: bool = False,
+    icon="#️⃣ ",
+) -> bool:
+    tags = Options(tags)
+
+    if not api.write(
+        object_name="_serverless_objects",
+        filename=f"{object_name}.yaml",
+        data=tags,
+        log=verbose,
+    ):
+        return False
+
+    for key, value in tags.items():
+        if not api.write(
+            object_name="_serverless_keys",
+            filename=f"{key}.yaml",
+            data={object_name: value},
+            log=verbose,
+        ):
+            return False
+
+        if log:
+            logger.info("{} {}.{}={}".format(icon, object_name, key, value))
+
+    return True

bluer_objects/mlflow/tags.py

@@ -7,13 +7,17 @@ from bluer_options.options import Options
 from bluer_options.logger import crash_report
 
 from bluer_objects import NAME
+from bluer_objects import env
+from bluer_objects.mlflow import serverless
 from bluer_objects.mlflow.objects import to_experiment_name, to_object_name
 from bluer_objects.logger import logger
 
 NAME = module.name(__file__, NAME)
 
 
-def create_filter_string(tags: str) -> str:
+def create_server_style_filter_string(
+    tags: Union[str, dict],
+) -> str:
     tags_options = Options(tags)
 
     # https://www.mlflow.org/docs/latest/search-experiments.html
@@ -25,7 +29,14 @@ def create_filter_string(tags: str) -> str:
 def get_tags(
     object_name: str,
     exclude_system_tags: bool = True,
+    verbose: bool = False,
 ) -> Tuple[bool, Dict[str, str]]:
+    if env.MLFLOW_IS_SERVERLESS:
+        return serverless.get_tags(
+            object_name,
+            verbose=verbose,
+        )
+
     experiment_name = to_experiment_name(object_name)
 
     try:
@@ -48,16 +59,45 @@ def get_tags(
 
 
 # https://www.mlflow.org/docs/latest/search-experiments.html
-def search(filter_string: str) -> List[str]:
-    client = MlflowClient()
-
-    return [
-        to_object_name(experiment.name)
-        for experiment in client.search_experiments(
+def search(
+    filter_string: Union[str, dict],
+    server_style: bool = False,
+    verbose: bool = False,
+) -> Tuple[bool, List[str]]:
+    if server_style and env.MLFLOW_IS_SERVERLESS:
+        logger.error("server_style search is not supported when mlflow is serverless.")
+        return False, []
+
+    if filter_string == "-":
+        filter_string = ""
+
+    if env.MLFLOW_IS_SERVERLESS:
+        return serverless.search(
             filter_string=filter_string,
-            view_type=ViewType.ALL,
+            log=verbose,
         )
-    ]
+
+    if not server_style:
+        filter_string = create_server_style_filter_string(filter_string)
+
+    client = MlflowClient()
+
+    success = False
+    output = []
+
+    try:
+        output = [
+            to_object_name(experiment.name)
+            for experiment in client.search_experiments(
+                filter_string=filter_string,
+                view_type=ViewType.ALL,
+            )
+        ]
+        success = True
+    except Exception as e:
+        logger.error(e)
+
+    return success, output
 
 
 def set_tags(
@@ -65,7 +105,17 @@ def set_tags(
     tags: Union[str, Dict[str, str]],
     log: bool = True,
     icon="#️⃣ ",
+    verbose: bool = False,
 ) -> bool:
+    if env.MLFLOW_IS_SERVERLESS:
+        return serverless.set_tags(
+            object_name,
+            tags=tags,
+            log=log,
+            verbose=verbose,
+            icon=icon,
+        )
+
     experiment_name = to_experiment_name(object_name)
 
     try:

bluer_objects/objects.py

@@ -69,6 +69,7 @@ def signature(info=None, object_name="."):
 def unique_object(
     prefix: str = "",
     include_time: bool = True,
+    log: bool = True,
 ):
     object_name = string.pretty_date(
         as_filename=True,
@@ -80,6 +81,7 @@ def unique_object(
 
     path.create(object_path(object_name))
 
-    logger.info(f"📂 {object_name}")
+    if log:
+        logger.info(f"📂 {object_name}")
 
     return object_name

bluer_objects/pdf/convert/convert.py

@@ -79,8 +79,7 @@ def convert(
             ):
                 return False
         elif file.extension(source_filename) in [
-            extension.split(".", 1)[1]
-            for extension in Image.registered_extensions().keys()
+            extension.split(".", 1)[1] for extension in Image.registered_extensions()
         ]:
             if not convert_image(
                 source_filename,

bluer_objects/tests/test_env.py

@@ -1,3 +1,8 @@
+from bluer_options.testing import (
+    are_01,
+    are_positive_ints,
+    are_nonempty_strs,
+)
 from bluer_ai.tests.test_env import test_bluer_ai_env
 
 from bluer_objects import env
@@ -14,14 +19,35 @@ def test_required_env():
 
 
 def test_bluer_objects_env():
-    assert env.ABCLI_MLFLOW_EXPERIMENT_PREFIX
-
-    assert env.S3_STORAGE_BUCKET
-    assert env.S3_PUBLIC_STORAGE_BUCKET
-
-    assert env.S3_STORAGE_ENDPOINT_URL
-    assert env.S3_STORAGE_AWS_ACCESS_KEY_ID
-    assert env.S3_STORAGE_AWS_SECRET_ACCESS_KEY
+    assert are_01(
+        [
+            env.MLFLOW_IS_SERVERLESS,
+        ]
+    )
+
+    assert are_nonempty_strs(
+        [
+            env.ABCLI_MLFLOW_EXPERIMENT_PREFIX,
+            env.BLUER_OBJECTS_DEFAULT_ASSETS_VOL,
+            env.BLUER_OBJECTS_TEST_OBJECT,
+            env.MLFLOW_DEPLOYMENT,
+            env.S3_PUBLIC_STORAGE_BUCKET,
+            env.S3_STORAGE_AWS_ACCESS_KEY_ID,
+            env.S3_STORAGE_AWS_SECRET_ACCESS_KEY,
+            env.S3_STORAGE_BUCKET,
+            env.S3_STORAGE_ENDPOINT_URL,
+            env.WEBDAV_HOSTNAME,
+            env.WEBDAV_LOGIN,
+            env.WEBDAV_PASSWORD,
+        ]
+    )
+
+    assert are_positive_ints(
+        [
+            env.MLFLOW_LOCK_WAIT_FOR_CLEARANCE,
+            env.MLFLOW_LOCK_WAIT_FOR_EXCLUSIVITY,
+        ]
+    )
 
     assert env.BLUER_OBJECTS_STORAGE_INTERFACE in [
         S3Interface.name,
@@ -29,17 +55,3 @@ def test_bluer_objects_env():
         WebDAVRequestInterface.name,
         WebDAVzipInterface.name,
     ]
-
-    assert env.MLFLOW_DEPLOYMENT
-
-    assert isinstance(env.MLFLOW_LOCK_WAIT_FOR_CLEARANCE, int)
-    assert env.MLFLOW_LOCK_WAIT_FOR_CLEARANCE > 0
-
-    assert isinstance(env.MLFLOW_LOCK_WAIT_FOR_EXCLUSIVITY, int)
-    assert env.MLFLOW_LOCK_WAIT_FOR_EXCLUSIVITY > 0
-
-    assert env.WEBDAV_HOSTNAME
-    assert env.WEBDAV_LOGIN
-    assert env.WEBDAV_PASSWORD
-
-    assert env.BLUER_OBJECTS_TEST_OBJECT

bluer_objects/tests/test_file_download.py

@@ -1,5 +1,7 @@
 import pytest
 
+from bluer_options.env import BLUER_AI_WEB_STATUS
+
 from bluer_objects import objects, file
 
 
@@ -10,6 +12,9 @@ from bluer_objects import objects, file
     ],
 )
 def test_file_download(url: str):
+    if BLUER_AI_WEB_STATUS != "online":
+        return
+
     object_name = objects.unique_object("test_file_download")
 
     filename = objects.path_of(

bluer_objects/tests/test_mlflow.py

@@ -1,10 +1,13 @@
 import pytest
 
-from bluer_options.options import Options
 from bluer_options import string
+from bluer_options.options import Options
+from bluer_options.testing import is_list_of_str
 
+from bluer_objects import env
 from bluer_objects.objects import unique_object
 from bluer_objects.mlflow import cache
+from bluer_objects.mlflow import create_server_style_filter_string
 from bluer_objects.mlflow import objects
 from bluer_objects.mlflow import tags
 from bluer_objects.mlflow import testing
@@ -19,14 +22,32 @@ def test_from_and_to_experiment_name():
 
 
 def test_mlflow_testing():
+    MLFLOW_IS_SERVERLESS = env.MLFLOW_IS_SERVERLESS
+    env.MLFLOW_IS_SERVERLESS = False
+
     assert testing.test()
 
+    env.MLFLOW_IS_SERVERLESS = MLFLOW_IS_SERVERLESS
+
 
 @pytest.mark.parametrize(
     ["tags_str"],
     [["x=1,y=2,z=3"]],
 )
-def test_mlflow_tag_set_get(tags_str: str):
+@pytest.mark.parametrize(
+    ["serverless"],
+    [
+        [False],
+        [True],
+    ],
+)
+def test_mlflow_tags_set_get(
+    tags_str: str,
+    serverless: bool,
+):
+    MLFLOW_IS_SERVERLESS = env.MLFLOW_IS_SERVERLESS
+    env.MLFLOW_IS_SERVERLESS = serverless
+
     object_name = unique_object("test_mlflow_tag_set_get")
 
     assert tags.set_tags(
@@ -42,6 +63,62 @@ def test_mlflow_tag_set_get(tags_str: str):
     for keyword, value in tags_option.items():
         assert tags_read[keyword] == value
 
+    env.MLFLOW_IS_SERVERLESS = MLFLOW_IS_SERVERLESS
+
+
+@pytest.mark.parametrize(
+    [
+        "filter_string",
+        "server_style",
+        "serverless",
+        "success_expected",
+    ],
+    [
+        [
+            "this=that,what,~who",
+            False,
+            True,
+            True,
+        ],
+        [
+            "this=that,what,~who",
+            False,
+            False,
+            True,
+        ],
+        [
+            'tags."this" = "that" and tags."what" = "True" and tags."who" = "False"',
+            True,
+            True,
+            False,
+        ],
+        [
+            'tags."this" = "that" and tags."what" = "True" and tags."who" = "False"',
+            True,
+            False,
+            True,
+        ],
+    ],
+)
+def test_mlflow_tags_search(
+    filter_string: str,
+    server_style: bool,
+    serverless: bool,
+    success_expected: bool,
+):
+    MLFLOW_IS_SERVERLESS = env.MLFLOW_IS_SERVERLESS
+    env.MLFLOW_IS_SERVERLESS = serverless
+
+    success, list_of_objects = tags.search(
+        filter_string,
+        server_style=server_style,
+    )
+
+    assert success == success_expected
+    assert is_list_of_str(list_of_objects)
+
+    env.MLFLOW_IS_SERVERLESS = MLFLOW_IS_SERVERLESS
+
 
 @pytest.mark.parametrize(
     ["keyword", "value"],
@@ -52,9 +129,41 @@ def test_mlflow_tag_set_get(tags_str: str):
         ]
     ],
 )
-def test_mlflow_cache_read_write(keyword: str, value: str):
+@pytest.mark.parametrize(
+    ["serverless"],
+    [
+        [True],
+        [False],
+    ],
+)
+def test_mlflow_cache_read_write(
+    keyword: str,
+    value: str,
+    serverless: bool,
+):
+    MLFLOW_IS_SERVERLESS = env.MLFLOW_IS_SERVERLESS
+    env.MLFLOW_IS_SERVERLESS = serverless
+
     assert cache.write(keyword, value)
 
     success, value_read = cache.read(keyword)
     assert success
     assert value_read == value
+
+    env.MLFLOW_IS_SERVERLESS = MLFLOW_IS_SERVERLESS
+
+
+@pytest.mark.parametrize(
+    ["filter_string", "expected_output"],
+    [
+        [
+            "this=that,what,~who",
+            'tags."this" = "that" and tags."what" = "True" and tags."who" = "False"',
+        ]
+    ],
+)
+def test_create_server_style_filter_string(
+    filter_string: str,
+    expected_output: str,
+):
+    assert create_server_style_filter_string(filter_string) == expected_output

bluer_objects/tests/test_web_is_accessible.py

@@ -1,4 +1,4 @@
-from bluer_objects.web.functions import is_accessible
+from bluer_ai.web.accessible import is_accessible
 from bluer_ai.env import abcli_is_github_workflow