PyPI - ob-metaflow - Versions diffs - 2.15.13.1__py2.py3-none-any.whl → 2.19.7.1rc0__py2.py3-none-any.whl - Mend

ob-metaflow 2.15.13.1py2.py3-none-any.whl → 2.19.7.1rc0py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (169) hide show

metaflow/__init__.py +10 -3
metaflow/_vendor/imghdr/__init__.py +186 -0
metaflow/_vendor/yaml/__init__.py +427 -0
metaflow/_vendor/yaml/composer.py +139 -0
metaflow/_vendor/yaml/constructor.py +748 -0
metaflow/_vendor/yaml/cyaml.py +101 -0
metaflow/_vendor/yaml/dumper.py +62 -0
metaflow/_vendor/yaml/emitter.py +1137 -0
metaflow/_vendor/yaml/error.py +75 -0
metaflow/_vendor/yaml/events.py +86 -0
metaflow/_vendor/yaml/loader.py +63 -0
metaflow/_vendor/yaml/nodes.py +49 -0
metaflow/_vendor/yaml/parser.py +589 -0
metaflow/_vendor/yaml/reader.py +185 -0
metaflow/_vendor/yaml/representer.py +389 -0
metaflow/_vendor/yaml/resolver.py +227 -0
metaflow/_vendor/yaml/scanner.py +1435 -0
metaflow/_vendor/yaml/serializer.py +111 -0
metaflow/_vendor/yaml/tokens.py +104 -0
metaflow/cards.py +4 -0
metaflow/cli.py +125 -21
metaflow/cli_components/init_cmd.py +1 -0
metaflow/cli_components/run_cmds.py +204 -40
metaflow/cli_components/step_cmd.py +160 -4
metaflow/client/__init__.py +1 -0
metaflow/client/core.py +198 -130
metaflow/client/filecache.py +59 -32
metaflow/cmd/code/__init__.py +2 -1
metaflow/cmd/develop/stub_generator.py +49 -18
metaflow/cmd/develop/stubs.py +9 -27
metaflow/cmd/make_wrapper.py +30 -0
metaflow/datastore/__init__.py +1 -0
metaflow/datastore/content_addressed_store.py +40 -9
metaflow/datastore/datastore_set.py +10 -1
metaflow/datastore/flow_datastore.py +124 -4
metaflow/datastore/spin_datastore.py +91 -0
metaflow/datastore/task_datastore.py +92 -6
metaflow/debug.py +5 -0
metaflow/decorators.py +331 -82
metaflow/extension_support/__init__.py +414 -356
metaflow/extension_support/_empty_file.py +2 -2
metaflow/flowspec.py +322 -82
metaflow/graph.py +178 -15
metaflow/includefile.py +25 -3
metaflow/lint.py +94 -3
metaflow/meta_files.py +13 -0
metaflow/metadata_provider/metadata.py +13 -2
metaflow/metaflow_config.py +66 -4
metaflow/metaflow_environment.py +91 -25
metaflow/metaflow_profile.py +18 -0
metaflow/metaflow_version.py +16 -1
metaflow/package/__init__.py +673 -0
metaflow/packaging_sys/__init__.py +880 -0
metaflow/packaging_sys/backend.py +128 -0
metaflow/packaging_sys/distribution_support.py +153 -0
metaflow/packaging_sys/tar_backend.py +99 -0
metaflow/packaging_sys/utils.py +54 -0
metaflow/packaging_sys/v1.py +527 -0
metaflow/parameters.py +6 -2
metaflow/plugins/__init__.py +6 -0
metaflow/plugins/airflow/airflow.py +11 -1
metaflow/plugins/airflow/airflow_cli.py +16 -5
metaflow/plugins/argo/argo_client.py +42 -20
metaflow/plugins/argo/argo_events.py +6 -6
metaflow/plugins/argo/argo_workflows.py +1023 -344
metaflow/plugins/argo/argo_workflows_cli.py +396 -94
metaflow/plugins/argo/argo_workflows_decorator.py +9 -0
metaflow/plugins/argo/argo_workflows_deployer_objects.py +75 -49
metaflow/plugins/argo/capture_error.py +5 -2
metaflow/plugins/argo/conditional_input_paths.py +35 -0
metaflow/plugins/argo/exit_hooks.py +209 -0
metaflow/plugins/argo/param_val.py +19 -0
metaflow/plugins/aws/aws_client.py +6 -0
metaflow/plugins/aws/aws_utils.py +33 -1
metaflow/plugins/aws/batch/batch.py +72 -5
metaflow/plugins/aws/batch/batch_cli.py +24 -3
metaflow/plugins/aws/batch/batch_decorator.py +57 -6
metaflow/plugins/aws/step_functions/step_functions.py +28 -3
metaflow/plugins/aws/step_functions/step_functions_cli.py +49 -4
metaflow/plugins/aws/step_functions/step_functions_deployer.py +3 -0
metaflow/plugins/aws/step_functions/step_functions_deployer_objects.py +30 -0
metaflow/plugins/cards/card_cli.py +20 -1
metaflow/plugins/cards/card_creator.py +24 -1
metaflow/plugins/cards/card_datastore.py +21 -49
metaflow/plugins/cards/card_decorator.py +58 -6
metaflow/plugins/cards/card_modules/basic.py +38 -9
metaflow/plugins/cards/card_modules/bundle.css +1 -1
metaflow/plugins/cards/card_modules/chevron/renderer.py +1 -1
metaflow/plugins/cards/card_modules/components.py +592 -3
metaflow/plugins/cards/card_modules/convert_to_native_type.py +34 -5
metaflow/plugins/cards/card_modules/json_viewer.py +232 -0
metaflow/plugins/cards/card_modules/main.css +1 -0
metaflow/plugins/cards/card_modules/main.js +56 -41
metaflow/plugins/cards/card_modules/test_cards.py +22 -6
metaflow/plugins/cards/component_serializer.py +1 -8
metaflow/plugins/cards/metadata.py +22 -0
metaflow/plugins/catch_decorator.py +9 -0
metaflow/plugins/datastores/local_storage.py +12 -6
metaflow/plugins/datastores/spin_storage.py +12 -0
metaflow/plugins/datatools/s3/s3.py +49 -17
metaflow/plugins/datatools/s3/s3op.py +113 -66
metaflow/plugins/env_escape/client_modules.py +102 -72
metaflow/plugins/events_decorator.py +127 -121
metaflow/plugins/exit_hook/__init__.py +0 -0
metaflow/plugins/exit_hook/exit_hook_decorator.py +46 -0
metaflow/plugins/exit_hook/exit_hook_script.py +52 -0
metaflow/plugins/kubernetes/kubernetes.py +12 -1
metaflow/plugins/kubernetes/kubernetes_cli.py +11 -0
metaflow/plugins/kubernetes/kubernetes_decorator.py +25 -6
metaflow/plugins/kubernetes/kubernetes_job.py +12 -4
metaflow/plugins/kubernetes/kubernetes_jobsets.py +31 -30
metaflow/plugins/metadata_providers/local.py +76 -82
metaflow/plugins/metadata_providers/service.py +13 -9
metaflow/plugins/metadata_providers/spin.py +16 -0
metaflow/plugins/package_cli.py +36 -24
metaflow/plugins/parallel_decorator.py +11 -2
metaflow/plugins/parsers.py +16 -0
metaflow/plugins/pypi/bootstrap.py +7 -1
metaflow/plugins/pypi/conda_decorator.py +41 -82
metaflow/plugins/pypi/conda_environment.py +14 -6
metaflow/plugins/pypi/micromamba.py +9 -1
metaflow/plugins/pypi/pip.py +41 -5
metaflow/plugins/pypi/pypi_decorator.py +4 -4
metaflow/plugins/pypi/utils.py +22 -0
metaflow/plugins/secrets/__init__.py +3 -0
metaflow/plugins/secrets/secrets_decorator.py +14 -178
metaflow/plugins/secrets/secrets_func.py +49 -0
metaflow/plugins/secrets/secrets_spec.py +101 -0
metaflow/plugins/secrets/utils.py +74 -0
metaflow/plugins/test_unbounded_foreach_decorator.py +2 -2
metaflow/plugins/timeout_decorator.py +0 -1
metaflow/plugins/uv/bootstrap.py +29 -1
metaflow/plugins/uv/uv_environment.py +5 -3
metaflow/pylint_wrapper.py +5 -1
metaflow/runner/click_api.py +79 -26
metaflow/runner/deployer.py +208 -6
metaflow/runner/deployer_impl.py +32 -12
metaflow/runner/metaflow_runner.py +266 -33
metaflow/runner/subprocess_manager.py +21 -1
metaflow/runner/utils.py +27 -16
metaflow/runtime.py +660 -66
metaflow/task.py +255 -26
metaflow/user_configs/config_options.py +33 -21
metaflow/user_configs/config_parameters.py +220 -58
metaflow/user_decorators/__init__.py +0 -0
metaflow/user_decorators/common.py +144 -0
metaflow/user_decorators/mutable_flow.py +512 -0
metaflow/user_decorators/mutable_step.py +424 -0
metaflow/user_decorators/user_flow_decorator.py +264 -0
metaflow/user_decorators/user_step_decorator.py +749 -0
metaflow/util.py +197 -7
metaflow/vendor.py +23 -7
metaflow/version.py +1 -1
{ob_metaflow-2.15.13.1.data → ob_metaflow-2.19.7.1rc0.data}/data/share/metaflow/devtools/Makefile +13 -2
{ob_metaflow-2.15.13.1.data → ob_metaflow-2.19.7.1rc0.data}/data/share/metaflow/devtools/Tiltfile +107 -7
{ob_metaflow-2.15.13.1.data → ob_metaflow-2.19.7.1rc0.data}/data/share/metaflow/devtools/pick_services.sh +1 -0
{ob_metaflow-2.15.13.1.dist-info → ob_metaflow-2.19.7.1rc0.dist-info}/METADATA +2 -3
{ob_metaflow-2.15.13.1.dist-info → ob_metaflow-2.19.7.1rc0.dist-info}/RECORD +162 -121
{ob_metaflow-2.15.13.1.dist-info → ob_metaflow-2.19.7.1rc0.dist-info}/WHEEL +1 -1
metaflow/_vendor/v3_5/__init__.py +0 -1
metaflow/_vendor/v3_5/importlib_metadata/__init__.py +0 -644
metaflow/_vendor/v3_5/importlib_metadata/_compat.py +0 -152
metaflow/_vendor/v3_5/zipp.py +0 -329
metaflow/info_file.py +0 -25
metaflow/package.py +0 -203
metaflow/user_configs/config_decorators.py +0 -568
{ob_metaflow-2.15.13.1.dist-info → ob_metaflow-2.19.7.1rc0.dist-info}/entry_points.txt +0 -0
{ob_metaflow-2.15.13.1.dist-info → ob_metaflow-2.19.7.1rc0.dist-info}/licenses/LICENSE +0 -0
{ob_metaflow-2.15.13.1.dist-info → ob_metaflow-2.19.7.1rc0.dist-info}/top_level.txt +0 -0

metaflow/plugins/cards/card_modules/test_cards.py CHANGED Viewed

@@ -34,7 +34,7 @@ class TestPathSpecCard(MetaflowCard):
 class TestEditableCard(MetaflowCard):
     type = "test_editable_card"
-    seperator = "$&#!!@*"
+    separator = "$&#!!@*"
     ALLOW_USER_COMPONENTS = True
@@ -42,13 +42,13 @@ class TestEditableCard(MetaflowCard):
         self._components = components
     def render(self, task):
-        return self.seperator.join([str(comp) for comp in self._components])
+        return self.separator.join([str(comp) for comp in self._components])
 class TestEditableCard2(MetaflowCard):
     type = "test_editable_card_2"
-    seperator = "$&#!!@*"
+    separator = "$&#!!@*"
     ALLOW_USER_COMPONENTS = True
@@ -56,19 +56,19 @@ class TestEditableCard2(MetaflowCard):
         self._components = components
     def render(self, task):
-        return self.seperator.join([str(comp) for comp in self._components])
+        return self.separator.join([str(comp) for comp in self._components])
 class TestNonEditableCard(MetaflowCard):
     type = "test_non_editable_card"
-    seperator = "$&#!!@*"
+    separator = "$&#!!@*"
     def __init__(self, components=[], **kwargs):
         self._components = components
     def render(self, task):
-        return self.seperator.join([str(comp) for comp in self._components])
+        return self.separator.join([str(comp) for comp in self._components])
 class TestMockCard(MetaflowCard):
@@ -213,3 +213,19 @@ class TestRefreshComponentCard(MetaflowCard):
         if task.finished:
             return "final"
         return "runtime-%s" % _component_values_to_hash(data["components"])
+class TestImageCard(MetaflowCard):
+    """Card that renders a tiny PNG using ``TaskToDict.parse_image``."""
+    type = "test_image_card"
+    def render(self, task):
+        from .convert_to_native_type import TaskToDict
+        import base64
+        png_bytes = base64.b64decode(
+            "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGBgAAAABQABRDE8UwAAAABJRU5ErkJggg=="
+        )
+        img_src = TaskToDict().parse_image(png_bytes)
+        return f"<html><img src='{img_src}' /></html>"

metaflow/plugins/cards/component_serializer.py CHANGED Viewed

@@ -57,15 +57,8 @@ class ComponentStore:
         The `_component_map` attribute is supposed to be a dictionary so that we can access the components by their ids.
         But we also want to maintain order in which components are inserted since all of these components are going to be visible on a UI.
         Since python3.6 dictionaries are ordered by default so we can use the default python `dict`.
-        For python3.5 and below we need to use an OrderedDict since `dict`'s are not ordered by default.
         """
-        python_version = int(platform.python_version_tuple()[0]) * 10 + int(
-            platform.python_version_tuple()[1]
-        )
-        if python_version < 36:
-            self._component_map = OrderedDict()
-        else:
-            self._component_map = {}
+        self._component_map = {}
     def __init__(self, logger, card_type=None, components=None, user_set_id=None):
         self._logger = logger

metaflow/plugins/cards/metadata.py ADDED Viewed

@@ -0,0 +1,22 @@
+import json
+from metaflow.metadata_provider import MetaDatum
+def _save_metadata(
+    metadata_provider,
+    run_id,
+    step_name,
+    task_id,
+    attempt_id,
+    card_uuid,
+    save_metadata,
+):
+    entries = [
+        MetaDatum(
+            field=card_uuid,
+            value=json.dumps(save_metadata),
+            type="card-info",
+            tags=["attempt_id:{0}".format(attempt_id)],
+        )
+    ]
+    metadata_provider.register_metadata(run_id, step_name, task_id, entries)

metaflow/plugins/catch_decorator.py CHANGED Viewed

@@ -52,6 +52,15 @@ class CatchDecorator(StepDecorator):
                 "split steps." % step
             )
+        # Do not support catch on switch steps for now.
+        # When applying @catch to a switch step, we can not guarantee that the flow attribute used for the switching condition gets properly recorded.
+        if graph[step].type == "split-switch":
+            raise MetaflowException(
+                "@catch is defined for the step *%s* "
+                "but @catch is not supported in conditional "
+                "switch steps." % step
+            )
     def _print_exception(self, step, flow):
         self.logger(head="@catch caught an exception from %s" % flow, timestamp=False)
         for line in traceback.format_exc().splitlines():

metaflow/plugins/datastores/local_storage.py CHANGED Viewed

@@ -1,24 +1,29 @@
 import json
 import os
-from metaflow.metaflow_config import DATASTORE_LOCAL_DIR, DATASTORE_SYSROOT_LOCAL
+from metaflow.metaflow_config import (
+    DATASTORE_LOCAL_DIR,
+    DATASTORE_SYSROOT_LOCAL,
+)
 from metaflow.datastore.datastore_storage import CloseAfterUse, DataStoreStorage
 class LocalStorage(DataStoreStorage):
     TYPE = "local"
     METADATA_DIR = "_meta"
+    DATASTORE_DIR = DATASTORE_LOCAL_DIR  # ".metaflow"
+    SYSROOT_VAR = DATASTORE_SYSROOT_LOCAL
     @classmethod
     def get_datastore_root_from_config(cls, echo, create_on_absent=True):
-        result = DATASTORE_SYSROOT_LOCAL
+        result = cls.SYSROOT_VAR
         if result is None:
             try:
                 # Python2
                 current_path = os.getcwdu()
             except:  # noqa E722
                 current_path = os.getcwd()
-            check_dir = os.path.join(current_path, DATASTORE_LOCAL_DIR)
+            check_dir = os.path.join(current_path, cls.DATASTORE_DIR)
             check_dir = os.path.realpath(check_dir)
             orig_path = check_dir
             top_level_reached = False
@@ -28,12 +33,13 @@ class LocalStorage(DataStoreStorage):
                     top_level_reached = True
                     break  # We are no longer making upward progress
                 current_path = new_path
-                check_dir = os.path.join(current_path, DATASTORE_LOCAL_DIR)
+                check_dir = os.path.join(current_path, cls.DATASTORE_DIR)
             if top_level_reached:
                 if create_on_absent:
                     # Could not find any directory to use so create a new one
                     echo(
-                        "Creating local datastore in current directory (%s)" % orig_path
+                        "Creating %s datastore in current directory (%s)"
+                        % (cls.TYPE, orig_path)
                     )
                     os.mkdir(orig_path)
                     result = orig_path
@@ -42,7 +48,7 @@ class LocalStorage(DataStoreStorage):
             else:
                 result = check_dir
         else:
-            result = os.path.join(result, DATASTORE_LOCAL_DIR)
+            result = os.path.join(result, cls.DATASTORE_DIR)
         return result
     @staticmethod

metaflow/plugins/datastores/spin_storage.py ADDED Viewed

@@ -0,0 +1,12 @@
+from metaflow.metaflow_config import (
+    DATASTORE_SPIN_LOCAL_DIR,
+    DATASTORE_SYSROOT_SPIN,
+)
+from metaflow.plugins.datastores.local_storage import LocalStorage
+class SpinStorage(LocalStorage):
+    TYPE = "spin"
+    METADATA_DIR = "_meta"
+    DATASTORE_DIR = DATASTORE_SPIN_LOCAL_DIR  # ".metaflow_spin"
+    SYSROOT_VAR = DATASTORE_SYSROOT_SPIN

metaflow/plugins/datatools/s3/s3.py CHANGED Viewed

@@ -18,6 +18,7 @@ from metaflow.metaflow_config import (
     DATATOOLS_S3ROOT,
     S3_RETRY_COUNT,
     S3_TRANSIENT_RETRY_COUNT,
+    S3_LOG_TRANSIENT_RETRIES,
     S3_SERVER_SIDE_ENCRYPTION,
     S3_WORKER_COUNT,
     TEMPDIR,
@@ -498,16 +499,18 @@ class S3(object):
     Parameters
     ----------
-    tmproot : str, default: '.'
+    tmproot : str, default '.'
         Where to store the temporary directory.
-    bucket : str, optional
+    bucket : str, optional, default None
         Override the bucket from `DATATOOLS_S3ROOT` when `run` is specified.
-    prefix : str, optional
+    prefix : str, optional, default None
         Override the path from `DATATOOLS_S3ROOT` when `run` is specified.
-    run : FlowSpec or Run, optional
+    run : FlowSpec or Run, optional, default None
         Derive path prefix from the current or a past run ID, e.g. S3(run=self).
-    s3root : str, optional
+    s3root : str, optional, default None
         If `run` is not specified, use this as the S3 prefix.
+    encryption : str, optional, default None
+        Server-side encryption to use when uploading objects to S3.
     """
     TYPE = "s3"
@@ -578,7 +581,13 @@ class S3(object):
         self._s3_inject_failures = kwargs.get(
             "inject_failure_rate", TEST_INJECT_RETRYABLE_FAILURES
         )
-        self._tmpdir = mkdtemp(dir=tmproot, prefix="metaflow.s3.")
+        # Storing tmproot, bucket, ... as members to allow easier reconstruction
+        # during JSON deserialization
+        self._tmproot = tmproot
+        self._bucket = bucket
+        self._prefix = prefix
+        self._run = run
+        self._tmpdir = mkdtemp(dir=self._tmproot, prefix="metaflow.s3.")
         self._encryption = encryption
     def __enter__(self) -> "S3":
@@ -629,7 +638,9 @@ class S3(object):
                     "Don't use absolute S3 URLs when the S3 client is "
                     "initialized with a prefix. URL: %s" % key
                 )
-            return os.path.join(self._s3root, key)
+            # Strip leading slashes to ensure os.path.join works correctly
+            # os.path.join discards the first argument if the second starts with '/'
+            return os.path.join(self._s3root, key.lstrip("/"))
         else:
             return self._s3root
@@ -1385,6 +1396,9 @@ class S3(object):
             except OSError as e:
                 if e.errno == errno.ENOSPC:
                     raise MetaflowS3InsufficientDiskSpace(str(e))
+            except MetaflowException as ex:
+                # Re-raise Metaflow exceptions (including TimeoutException)
+                raise
             except Exception as ex:
                 error = str(ex)
             if tmp:
@@ -1757,17 +1771,35 @@ class S3(object):
                 # due to a transient failure so we try again.
                 transient_retry_count += 1
                 total_ok_count += last_ok_count
-                print(
-                    "Transient S3 failure (attempt #%d) -- total success: %d, "
-                    "last attempt %d/%d -- remaining: %d"
-                    % (
-                        transient_retry_count,
-                        total_ok_count,
-                        last_ok_count,
-                        last_ok_count + last_retry_count,
-                        len(pending_retries),
+                if S3_LOG_TRANSIENT_RETRIES:
+                    # Extract transient error type from pending retry lines
+                    error_info = ""
+                    if pending_retries:
+                        try:
+                            # Parse the first line to get transient error type
+                            first_retry = json.loads(
+                                pending_retries[0].decode("utf-8").strip()
+                            )
+                            if "transient_error_type" in first_retry:
+                                error_info = (
+                                    " (%s)" % first_retry["transient_error_type"]
+                                )
+                        except (json.JSONDecodeError, IndexError, KeyError):
+                            pass
+                    print(
+                        "Transient S3 failure (attempt #%d) -- total success: %d, "
+                        "last attempt %d/%d -- remaining: %d%s"
+                        % (
+                            transient_retry_count,
+                            total_ok_count,
+                            last_ok_count,
+                            last_ok_count + last_retry_count,
+                            len(pending_retries),
+                            error_info,
+                        )
                     )
-                )
                 if inject_failures == 0:
                     # Don't sleep when we are "faking" the failures
                     self._jitter_sleep(transient_retry_count)

metaflow/plugins/datatools/s3/s3op.py CHANGED Viewed

@@ -50,6 +50,7 @@ import metaflow.tracing as tracing
 from metaflow.metaflow_config import (
     S3_WORKER_COUNT,
 )
+from metaflow.exception import MetaflowException
 DOWNLOAD_FILE_THRESHOLD = 2 * TransferConfig().multipart_threshold
 DOWNLOAD_MAX_CHUNK = 2 * 1024 * 1024 * 1024 - 1
@@ -131,7 +132,7 @@ def normalize_client_error(err):
     except ValueError:
         if error_code in ("AccessDenied", "AllAccessDisabled", "InvalidAccessKeyId"):
             return 403
-        if error_code == "NoSuchKey":
+        if error_code in ("NoSuchKey", "NoSuchBucket"):
             return 404
         if error_code == "InvalidRange":
             return 416
@@ -284,14 +285,23 @@ def worker(result_file_name, queue, mode, s3config):
                                     "%d %d\n" % (idx, -ERROR_OUT_OF_DISK_SPACE)
                                 )
                             else:
-                                result_file.write("%d %d\n" % (idx, -ERROR_TRANSIENT))
+                                result_file.write(
+                                    "%d %d %s\n" % (idx, -ERROR_TRANSIENT, "OSError")
+                                )
                             result_file.flush()
                             continue
+                    except MetaflowException:
+                        # Re-raise Metaflow exceptions (including TimeoutException)
+                        tmp.close()
+                        os.unlink(tmp.name)
+                        raise
                     except (SSLError, Exception) as e:
                         tmp.close()
                         os.unlink(tmp.name)
                         # assume anything else is transient
-                        result_file.write("%d %d\n" % (idx, -ERROR_TRANSIENT))
+                        result_file.write(
+                            "%d %d %s\n" % (idx, -ERROR_TRANSIENT, type(e).__name__)
+                        )
                         result_file.flush()
                         continue
                     # If we need the metadata, get it and write it out
@@ -357,9 +367,14 @@ def worker(result_file_name, queue, mode, s3config):
                                 err = convert_to_client_error(e)
                                 handle_client_error(err, idx, result_file)
                                 continue
+                        except MetaflowException:
+                            # Re-raise Metaflow exceptions (including TimeoutException)
+                            raise
                         except (SSLError, Exception) as e:
                             # assume anything else is transient
-                            result_file.write("%d %d\n" % (idx, -ERROR_TRANSIENT))
+                            result_file.write(
+                                "%d %d %s\n" % (idx, -ERROR_TRANSIENT, type(e).__name__)
+                            )
                             result_file.flush()
                             continue
         except:
@@ -385,7 +400,13 @@ def convert_to_client_error(e):
 def handle_client_error(err, idx, result_file):
+    # Handle all MetaflowExceptions as fatal
+    if isinstance(err, MetaflowException):
+        raise err
     error_code = normalize_client_error(err)
+    original_error_code = err.response["Error"]["Code"]
     if error_code == 404:
         result_file.write("%d %d\n" % (idx, -ERROR_URL_NOT_FOUND))
         result_file.flush()
@@ -393,13 +414,12 @@ def handle_client_error(err, idx, result_file):
         result_file.write("%d %d\n" % (idx, -ERROR_URL_ACCESS_DENIED))
         result_file.flush()
     elif error_code == 503:
-        result_file.write("%d %d\n" % (idx, -ERROR_TRANSIENT))
+        result_file.write("%d %d %s\n" % (idx, -ERROR_TRANSIENT, original_error_code))
         result_file.flush()
     else:
         # optimistically assume it is a transient error
-        result_file.write("%d %d\n" % (idx, -ERROR_TRANSIENT))
+        result_file.write("%d %d %s\n" % (idx, -ERROR_TRANSIENT, original_error_code))
         result_file.flush()
-    # TODO specific error message for out of disk space
 def start_workers(mode, urls, num_workers, inject_failure, s3config):
@@ -411,6 +431,7 @@ def start_workers(mode, urls, num_workers, inject_failure, s3config):
     random.seed()
     sz_results = []
+    transient_error_type = None
     # 1. push sources and destinations to the queue
     # We only push if we don't inject a failure; otherwise, we already set the sz_results
     # appropriately with the result of the injected failure.
@@ -465,13 +486,19 @@ def start_workers(mode, urls, num_workers, inject_failure, s3config):
                     # Read the output file if all went well
                     with open(out_path, "r") as out_file:
                         for line in out_file:
-                            line_split = line.split(" ")
-                            sz_results[int(line_split[0])] = int(line_split[1])
+                            line_split = line.split(" ", 2)
+                            idx = int(line_split[0])
+                            size = int(line_split[1])
+                            sz_results[idx] = size
+                            # For transient errors, store the transient error type (should be the same for all)
+                            if size == -ERROR_TRANSIENT and len(line_split) > 2:
+                                transient_error_type = line_split[2].strip()
                 else:
                     # Put this process back in the processes to check
                     new_procs[proc] = out_path
             procs = new_procs
-    return sz_results
+    return sz_results, transient_error_type
 def process_urls(mode, urls, verbose, inject_failure, num_workers, s3config):
@@ -480,7 +507,9 @@ def process_urls(mode, urls, verbose, inject_failure, num_workers, s3config):
         print("%sing %d files.." % (mode.capitalize(), len(urls)), file=sys.stderr)
     start = time.time()
-    sz_results = start_workers(mode, urls, num_workers, inject_failure, s3config)
+    sz_results, transient_error_type = start_workers(
+        mode, urls, num_workers, inject_failure, s3config
+    )
     end = time.time()
     if verbose:
@@ -497,7 +526,7 @@ def process_urls(mode, urls, verbose, inject_failure, num_workers, s3config):
             ),
             file=sys.stderr,
         )
-    return sz_results
+    return sz_results, transient_error_type
 # Utility functions
@@ -582,11 +611,12 @@ class S3Ops(object):
                 # - the trailing slash is significant in S3
                 if "Contents" in page:
                     for key in page.get("Contents", []):
-                        url = url_base + key["Key"]
+                        key_path = key["Key"].lstrip("/")
+                        url = url_base + key_path
                         urlobj = S3Url(
                             url=url,
                             bucket=prefix_url.bucket,
-                            path=key["Key"],
+                            path=key_path,
                             local=generate_local_path(url),
                             prefix=prefix_url.url,
                         )
@@ -694,9 +724,21 @@ def generate_local_path(url, range="whole", suffix=None):
     quoted = url_quote(url)
     fname = quoted.split(b"/")[-1].replace(b".", b"_").replace(b"-", b"_")
     sha = sha1(quoted).hexdigest()
+    # Truncate fname to ensure the final filename doesn't exceed filesystem limits.
+    # Most filesystems have a 255 character limit. The structure is:
+    # <40-char-sha>-<fname>-<range>[-<suffix>]
+    # We need to leave room for: sha (40) + hyphens (2-3) + range (~10) + suffix (~10)
+    # This leaves roughly 190 characters for fname. We use 150 to be safe.
+    fname_decoded = fname.decode("utf-8")
+    max_fname_len = 150
+    if len(fname_decoded) > max_fname_len:
+        # Truncate and add an ellipsis to indicate truncation
+        fname_decoded = fname_decoded[:max_fname_len] + "..."
     if suffix:
-        return "-".join((sha, fname.decode("utf-8"), range, suffix))
-    return "-".join((sha, fname.decode("utf-8"), range))
+        return "-".join((sha, fname_decoded, range, suffix))
+    return "-".join((sha, fname_decoded, range))
 def parallel_op(op, lst, num_workers):
@@ -833,11 +875,16 @@ def lst(
     urllist = []
     to_iterate, _ = _populate_prefixes(prefixes, inputs)
     for _, prefix, url, _ in to_iterate:
-        src = urlparse(url)
+        src = urlparse(url, allow_fragments=False)
+        # We always consider the path being passed in to be a directory path so
+        # we add a trailing slash to the path if it doesn't already have one.
+        path_with_slash = src.path.lstrip("/")
+        if not path_with_slash.endswith("/"):
+            path_with_slash += "/"
         url = S3Url(
             url=url,
             bucket=src.netloc,
-            path=src.path.lstrip("/"),
+            path=path_with_slash,
             local=None,
             prefix=prefix,
         )
@@ -939,7 +986,7 @@ def put(
                 yield input_line_idx, local, url, content_type, metadata, encryption
     def _make_url(idx, local, user_url, content_type, metadata, encryption):
-        src = urlparse(user_url)
+        src = urlparse(user_url, allow_fragments=False)
         url = S3Url(
             url=user_url,
             bucket=src.netloc,
@@ -967,7 +1014,7 @@ def put(
     ul_op = "upload"
     if not overwrite:
         ul_op = "info_upload"
-    sz_results = process_urls(
+    sz_results, transient_error_type = process_urls(
         ul_op, urls, verbose, inject_failure, num_workers, s3config
     )
     retry_lines = []
@@ -985,19 +1032,17 @@ def put(
         elif listing and sz == 0:
             out_lines.append(format_result_line(url.idx, url.url) + "\n")
         elif sz == -ERROR_TRANSIENT:
-            retry_lines.append(
-                json.dumps(
-                    {
-                        "idx": url.idx,
-                        "url": url.url,
-                        "local": url.local,
-                        "content_type": url.content_type,
-                        "metadata": url.metadata,
-                        "encryption": url.encryption,
-                    }
-                )
-                + "\n"
-            )
+            retry_data = {
+                "idx": url.idx,
+                "url": url.url,
+                "local": url.local,
+                "content_type": url.content_type,
+                "metadata": url.metadata,
+                "encryption": url.encryption,
+            }
+            if transient_error_type:
+                retry_data["transient_error_type"] = transient_error_type
+            retry_lines.append(json.dumps(retry_data) + "\n")
             # Output something to get a total count the first time around
             if not is_transient_retry:
                 out_lines.append("%d %s\n" % (url.idx, TRANSIENT_RETRY_LINE_CONTENT))
@@ -1035,22 +1080,21 @@ def _populate_prefixes(prefixes, inputs):
             for idx, l in enumerate(f, start=len(prefixes)):
                 s = l.split(b" ")
                 if len(s) == 1:
+                    # User input format: <url>
                     url = url_unquote(s[0].strip())
                     prefixes.append((idx, url, url, None))
                 elif len(s) == 2:
+                    # User input format: <url> <range>
                     url = url_unquote(s[0].strip())
                     prefixes.append((idx, url, url, url_unquote(s[1].strip())))
-                else:
+                elif len(s) in (4, 5):
+                    # Retry format: <idx> <prefix> <url> <range> [<transient_error_type>]
+                    # The transient_error_type (5th field) is optional and only used for logging.
+                    # Lines with other field counts (e.g., 3) are silently ignored as invalid.
                     is_transient_retry = True
-                    if len(s) == 3:
-                        prefix = url = url_unquote(s[1].strip())
-                        range_info = url_unquote(s[2].strip())
-                    else:
-                        # Special case when we have both prefix and URL -- this is
-                        # used in recursive gets for example
-                        prefix = url_unquote(s[1].strip())
-                        url = url_unquote(s[2].strip())
-                        range_info = url_unquote(s[3].strip())
+                    prefix = url_unquote(s[1].strip())
+                    url = url_unquote(s[2].strip())
+                    range_info = url_unquote(s[3].strip())
                     if range_info == "<norange>":
                         range_info = None
                     prefixes.append(
@@ -1114,7 +1158,7 @@ def get(
     urllist = []
     to_iterate, is_transient_retry = _populate_prefixes(prefixes, inputs)
     for idx, prefix, url, r in to_iterate:
-        src = urlparse(url)
+        src = urlparse(url, allow_fragments=False)
         url = S3Url(
             url=url,
             bucket=src.netloc,
@@ -1161,7 +1205,7 @@ def get(
     # exclude the non-existent files from loading
     to_load = [url for url, size in urls if size is not None]
-    sz_results = process_urls(
+    sz_results, transient_error_type = process_urls(
         dl_op, to_load, verbose, inject_failure, num_workers, s3config
     )
     # We check if there is any access denied
@@ -1197,21 +1241,19 @@ def get(
                 break
             out_lines.append(format_result_line(url.idx, url.url) + "\n")
         elif sz == -ERROR_TRANSIENT:
-            retry_lines.append(
-                " ".join(
-                    [
-                        str(url.idx),
-                        url_quote(url.prefix).decode(encoding="utf-8"),
-                        url_quote(url.url).decode(encoding="utf-8"),
-                        (
-                            url_quote(url.range).decode(encoding="utf-8")
-                            if url.range
-                            else "<norange>"
-                        ),
-                    ]
-                )
-                + "\n"
-            )
+            retry_line_parts = [
+                str(url.idx),
+                url_quote(url.prefix).decode(encoding="utf-8"),
+                url_quote(url.url).decode(encoding="utf-8"),
+                (
+                    url_quote(url.range).decode(encoding="utf-8")
+                    if url.range
+                    else "<norange>"
+                ),
+            ]
+            if transient_error_type:
+                retry_line_parts.append(transient_error_type)
+            retry_lines.append(" ".join(retry_line_parts) + "\n")
             # First time around, we output something to indicate the total length
             if not is_transient_retry:
                 out_lines.append("%d %s\n" % (url.idx, TRANSIENT_RETRY_LINE_CONTENT))
@@ -1263,7 +1305,7 @@ def info(
     urllist = []
     to_iterate, is_transient_retry = _populate_prefixes(prefixes, inputs)
     for idx, prefix, url, _ in to_iterate:
-        src = urlparse(url)
+        src = urlparse(url, allow_fragments=False)
         url = S3Url(
             url=url,
             bucket=src.netloc,
@@ -1277,7 +1319,7 @@ def info(
             exit(ERROR_INVALID_URL, url)
         urllist.append(url)
-    sz_results = process_urls(
+    sz_results, transient_error_type = process_urls(
         "info", urllist, verbose, inject_failure, num_workers, s3config
     )
@@ -1290,10 +1332,15 @@ def info(
                 format_result_line(url.idx, url.prefix, url.url, url.local) + "\n"
             )
         else:
-            retry_lines.append(
-                "%d %s <norange>\n"
-                % (url.idx, url_quote(url.url).decode(encoding="utf-8"))
-            )
+            retry_line_parts = [
+                str(url.idx),
+                url_quote(url.prefix).decode(encoding="utf-8"),
+                url_quote(url.url).decode(encoding="utf-8"),
+                "<norange>",
+            ]
+            if transient_error_type:
+                retry_line_parts.append(transient_error_type)
+            retry_lines.append(" ".join(retry_line_parts) + "\n")
             if not is_transient_retry:
                 out_lines.append("%d %s\n" % (url.idx, TRANSIENT_RETRY_LINE_CONTENT))

ob-metaflow 2.15.13.1__py2.py3-none-any.whl → 2.19.7.1rc0__py2.py3-none-any.whl

ob-metaflow 2.15.13.1py2.py3-none-any.whl → 2.19.7.1rc0py2.py3-none-any.whl