ob-metaflow 2.11.13.1__py2.py3-none-any.whl → 2.19.7.1rc0__py2.py3-none-any.whl

metaflow/util.py

@@ -4,12 +4,13 @@ import sys
 import tempfile
 import zlib
 import base64
+import re
+
 from functools import wraps
 from io import BytesIO
 from itertools import takewhile
-import re
+from typing import Dict, Any, Tuple, Optional, List, Generator
 
-from metaflow.exception import MetaflowUnknownUser, MetaflowInternalError
 
 try:
     # python2
@@ -51,21 +52,6 @@ except NameError:
     from shlex import quote as _quote
 
 
-from typing import NamedTuple
-
-
-def namedtuple_with_defaults(typename, field_descr, defaults=()):
-    T = NamedTuple(typename, field_descr)
-    T.__new__.__defaults__ = tuple(defaults)
-
-    # Adding the following to ensure the named tuple can be (un)pickled correctly.
-    import __main__
-
-    setattr(__main__, T.__name__, T)
-    T.__module__ = "__main__"
-    return T
-
-
 class TempDir(object):
     # Provide a temporary directory since Python 2.7 does not have it inbuilt
     def __enter__(self):
@@ -177,6 +163,8 @@ def get_username():
 
 
 def resolve_identity_as_tuple():
+    from metaflow.exception import MetaflowUnknownUser
+
     prod_token = os.environ.get("METAFLOW_PRODUCTION_TOKEN")
     if prod_token:
         return "production", prod_token
@@ -192,6 +180,119 @@ def resolve_identity():
     return "%s:%s" % (identity_type, identity_value)
 
 
+def parse_spin_pathspec(pathspec: str, flow_name: str) -> Tuple:
+    """
+    Parse various pathspec formats for the spin command.
+
+    Parameters
+    ----------
+    pathspec : str
+        The pathspec string in one of the following formats:
+        - step_name (e.g., 'start')
+        - run_id/step_name (e.g., '221165/start')
+        - run_id/step_name/task_id (e.g., '221165/start/1350987')
+        - flow_name/run_id/step_name (e.g., 'ScalableFlow/221165/start')
+        - flow_name/run_id/step_name/task_id (e.g., 'ScalableFlow/221165/start/1350987')
+    flow_name : str
+        The name of the current flow.
+
+    Returns
+    -------
+    Tuple
+        A tuple of (step_name, full_pathspec_or_none)
+
+    Raises
+    ------
+    CommandException
+        If the pathspec format is invalid or flow name doesn't match.
+    """
+    from .exception import CommandException
+
+    parts = pathspec.split("/")
+
+    if len(parts) == 1:
+        # Just step name: 'start'
+        step_name = parts[0]
+        parsed_pathspec = None
+    elif len(parts) == 2:
+        # run_id/step_name: '221165/start'
+        run_id, step_name = parts
+        parsed_pathspec = f"{flow_name}/{run_id}/{step_name}"
+    elif len(parts) == 3:
+        # Could be run_id/step_name/task_id or flow_name/run_id/step_name
+        if parts[0] == flow_name:
+            # flow_name/run_id/step_name
+            _, run_id, step_name = parts
+            parsed_pathspec = f"{flow_name}/{run_id}/{step_name}"
+        else:
+            # run_id/step_name/task_id
+            run_id, step_name, task_id = parts
+            parsed_pathspec = f"{flow_name}/{run_id}/{step_name}/{task_id}"
+    elif len(parts) == 4:
+        # flow_name/run_id/step_name/task_id
+        parsed_flow_name, run_id, step_name, task_id = parts
+        if parsed_flow_name != flow_name:
+            raise CommandException(
+                f"Flow name '{parsed_flow_name}' in pathspec does not match current flow '{flow_name}'."
+            )
+        parsed_pathspec = pathspec
+    else:
+        raise CommandException(
+            f"Invalid pathspec format: '{pathspec}'. \n"
+            "Expected formats:\n"
+            "  - step_name (e.g., 'start')\n"
+            "  - run_id/step_name (e.g., '221165/start')\n"
+            "  - run_id/step_name/task_id (e.g., '221165/start/1350987')\n"
+            "  - flow_name/run_id/step_name (e.g., 'ScalableFlow/221165/start')\n"
+            "  - flow_name/run_id/step_name/task_id (e.g., 'ScalableFlow/221165/start/1350987')"
+        )
+
+    return step_name, parsed_pathspec
+
+
+def get_latest_task_pathspec(
+    flow_name: str, step_name: str, run_id: str = None
+) -> "metaflow.Task":
+    """
+    Returns a task pathspec from the latest run (or specified run) of the flow for the queried step.
+    If the queried step has several tasks, the task pathspec of the first task is returned.
+
+    Parameters
+    ----------
+    flow_name : str
+        The name of the flow.
+    step_name : str
+        The name of the step.
+    run_id : str, optional
+        The run ID to use. If None, uses the latest run.
+
+    Returns
+    -------
+    Task
+        A Metaflow Task instance containing the latest task for the queried step.
+
+    Raises
+    ------
+    MetaflowNotFound
+        If no task or run is found for the queried step.
+    """
+    from metaflow import Flow, Step
+    from metaflow.exception import MetaflowNotFound
+
+    if not run_id:
+        flow = Flow(flow_name)
+        run = flow.latest_run
+        if run is None:
+            raise MetaflowNotFound(f"No run found for flow {flow_name}")
+        run_id = run.id
+
+    try:
+        task = Step(f"{flow_name}/{run_id}/{step_name}").task
+        return task
+    except:
+        raise MetaflowNotFound(f"No task found for step {step_name} in run {run_id}")
+
+
 def get_latest_run_id(echo, flow_name):
     from metaflow.plugins.datastores.local_storage import LocalStorage
 
@@ -251,6 +352,8 @@ def get_object_package_version(obj):
 
 
 def compress_list(lst, separator=",", rangedelim=":", zlibmarker="!", zlibmin=500):
+    from metaflow.exception import MetaflowInternalError
+
     bad_items = [x for x in lst if separator in x or rangedelim in x or zlibmarker in x]
     if bad_items:
         raise MetaflowInternalError(
@@ -311,6 +414,9 @@ def get_metaflow_root():
 
 
 def dict_to_cli_options(params):
+    # Prevent circular imports
+    from .user_configs.config_options import ConfigInput
+
     for k, v in params.items():
         # Omit boolean options set to false or None, but preserve options with an empty
         # string argument.
@@ -319,6 +425,20 @@ def dict_to_cli_options(params):
             # keyword in Python, so we call it 'decospecs' in click args
             if k == "decospecs":
                 k = "with"
+            if k in ("config", "config_value"):
+                # Special handling here since we gather them all in one option but actually
+                # need to send them one at a time using --config-value <name> kv.<name>
+                # Note it can be either config or config_value depending
+                # on click processing order.
+                for config_name in v.keys():
+                    yield "--config-value"
+                    yield to_unicode(config_name)
+                    yield to_unicode(ConfigInput.make_key_name(config_name))
+                continue
+            if k == "local_config_file":
+                # Skip this value -- it should only be used locally and never when
+                # forming another command line
+                continue
             k = k.replace("_", "-")
             v = v if isinstance(v, (list, tuple, set)) else [v]
             for value in v:
@@ -397,9 +517,9 @@ def to_camelcase(obj):
     if isinstance(obj, dict):
         res = obj.__class__()
         for k in obj:
-            res[
-                re.sub(r"(?!^)_([a-zA-Z])", lambda x: x.group(1).upper(), k)
-            ] = to_camelcase(obj[k])
+            res[re.sub(r"(?!^)_([a-zA-Z])", lambda x: x.group(1).upper(), k)] = (
+                to_camelcase(obj[k])
+            )
     elif isinstance(obj, (list, set, tuple)):
         res = obj.__class__(to_camelcase(v) for v in obj)
     else:
@@ -416,9 +536,9 @@ def to_pascalcase(obj):
     if isinstance(obj, dict):
         res = obj.__class__()
         for k in obj:
-            res[
-                re.sub("([a-zA-Z])", lambda x: x.groups()[0].upper(), k, 1)
-            ] = to_pascalcase(obj[k])
+            res[re.sub("([a-zA-Z])", lambda x: x.groups()[0].upper(), k, count=1)] = (
+                to_pascalcase(obj[k])
+            )
     elif isinstance(obj, (list, set, tuple)):
         res = obj.__class__(to_pascalcase(v) for v in obj)
     else:
@@ -441,7 +561,103 @@ def tar_safe_extract(tar, path=".", members=None, *, numeric_owner=False):
     tar.extractall(path, members, numeric_owner=numeric_owner)
 
 
-if sys.version_info[:2] > (3, 5):
-    from metaflow._vendor.packaging.version import parse as version_parse
-else:
-    from distutils.version import LooseVersion as version_parse
+def to_pod(value):
+    """
+    Convert a python object to plain-old-data (POD) format.
+
+    Parameters
+    ----------
+    value : Any
+        Value to convert to POD format. The value can be a string, number, list,
+        dictionary, or a nested structure of these types.
+    """
+    # Prevent circular imports
+    from metaflow.parameters import DeployTimeField
+
+    if isinstance(value, (str, int, float)):
+        return value
+    if isinstance(value, dict):
+        return {to_pod(k): to_pod(v) for k, v in value.items()}
+    if isinstance(value, (list, set, tuple)):
+        return [to_pod(v) for v in value]
+    if isinstance(value, DeployTimeField):
+        return value.print_representation
+    return str(value)
+
+
+from metaflow._vendor.packaging.version import parse as version_parse
+
+
+def read_artifacts_module(file_path: str) -> Dict[str, Any]:
+    """
+    Read a Python module from the given file path and return its ARTIFACTS variable.
+
+    Parameters
+    ----------
+    file_path : str
+        The path to the Python file containing the ARTIFACTS variable.
+
+    Returns
+    -------
+    Dict[str, Any]
+        A dictionary containing the ARTIFACTS variable from the module.
+
+    Raises
+    -------
+    MetaflowInternalError
+        If the file cannot be read or does not contain the ARTIFACTS variable.
+    """
+    import importlib.util
+    import os
+
+    try:
+        module_name = os.path.splitext(os.path.basename(file_path))[0]
+        spec = importlib.util.spec_from_file_location(module_name, file_path)
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+        variables = vars(module)
+        if "ARTIFACTS" not in variables:
+            raise MetaflowInternalError(
+                f"Module {file_path} does not contain ARTIFACTS variable"
+            )
+        return variables.get("ARTIFACTS")
+    except Exception as e:
+        raise MetaflowInternalError(f"Error reading file {file_path}") from e
+
+
+# this is os.walk(follow_symlinks=True) with cycle detection
+def walk_without_cycles(
+    top_root: str,
+    exclude_dirs: Optional[List[str]] = None,
+) -> Generator[Tuple[str, List[str], List[str]], None, None]:
+    seen = set()
+
+    default_skip_dirs = ["__pycache__"]
+
+    def _recurse(root, skip_dirs):
+        for parent, dirs, files in os.walk(root):
+            dirs[:] = [d for d in dirs if d not in skip_dirs]
+            for d in dirs:
+                path = os.path.join(parent, d)
+                if os.path.islink(path):
+                    # Breaking loops: never follow the same symlink twice
+                    #
+                    # NOTE: this also means that links to sibling links are
+                    # not followed. In this case:
+                    #
+                    #   x -> y
+                    #   y -> oo
+                    #   oo/real_file
+                    #
+                    # real_file is only included twice, not three times
+                    reallink = os.path.realpath(path)
+                    if reallink not in seen:
+                        seen.add(reallink)
+                        for x in _recurse(path, default_skip_dirs):
+                            yield x
+            yield parent, dirs, files
+
+    skip_dirs = set(default_skip_dirs + (exclude_dirs or []))
+    for x in _recurse(top_root, skip_dirs):
+        skip_dirs = default_skip_dirs
+        yield x

metaflow/vendor.py

@@ -11,7 +11,6 @@ WHITELIST = {
     "README.txt",
     "__init__.py",
     "vendor_any.txt",
-    "vendor_v3_5.txt",
     "vendor_v3_6.txt",
     "vendor_v3_7.txt",
     "pip.LICENSE",
@@ -64,14 +63,29 @@ def find_vendored_libs(vendor_dir, whitelist, whitelist_dirs):
     return vendored_libs, paths
 
 
-def fetch_licenses(*info_dir, vendor_dir):
-    for file in chain.from_iterable(map(iter_subtree, info_dir)):
-        if "LICENSE" in file.name:
-            library = file.parent.name.split("-")[0]
-            shutil.copy(file, vendor_dir / ("%s.LICENSE" % library))
-        else:
+def fetch_licenses(*info_dirs, vendor_dir):
+    for dist_info in info_dirs:
+        metadata_file = dist_info / "METADATA"
+        if not metadata_file.exists():
+            continue
+
+        project_name = None
+        for line in metadata_file.read_text("utf-8").splitlines():
+            if line.startswith("Name: "):
+                project_name = line.split("Name: ", 1)[1].strip()
+                break
+        if not project_name:
             continue
 
+        for item in dist_info.iterdir():
+            if item.is_file() and re.search(r"(LICENSE|COPYING)", item.name, re.I):
+                shutil.copy(item, vendor_dir / f"{project_name}.LICENSE")
+            elif item.is_dir() and item.name.lower() == "licenses":
+                for license_file in item.iterdir():
+                    if license_file.is_file():
+                        dest_name = f"{project_name}.{license_file.name}"
+                        shutil.copy(license_file, vendor_dir / dest_name)
+
 
 def vendor(vendor_dir):
     # remove everything
@@ -109,6 +123,8 @@ def vendor(vendor_dir):
                 "-r",
                 "_vendor/vendor_%s.txt" % subdir,
                 "--no-compile",
+                "--no-binary",
+                ":all:",
             ]
         )
 

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.11.13.1"
+metaflow_version = "2.19.7.1rc0"