ob-metaflow 2.12.36.2__py2.py3-none-any.whl → 2.12.36.3__py2.py3-none-any.whl

metaflow/plugins/pypi/pypi_decorator.py

@@ -25,10 +25,9 @@ class PyPIStepDecorator(StepDecorator):
     defaults = {"packages": {}, "python": None, "disabled": None}  # wheels
 
     def __init__(self, attributes=None, statically_defined=False):
-        self._attributes_with_user_values = (
-            set(attributes.keys()) if attributes is not None else set()
+        self._user_defined_attributes = (
+            attributes.copy() if attributes is not None else {}
         )
-
         super().__init__(attributes, statically_defined)
 
     def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
@@ -43,9 +42,10 @@ class PyPIStepDecorator(StepDecorator):
         if "pypi_base" in self.flow._flow_decorators:
             pypi_base = self.flow._flow_decorators["pypi_base"][0]
             super_attributes = pypi_base.attributes
-            self._attributes_with_user_values.update(
-                pypi_base._attributes_with_user_values
-            )
+            self._user_defined_attributes = {
+                **self._user_defined_attributes,
+                **pypi_base._user_defined_attributes,
+            }
             self.attributes["packages"] = {
                 **super_attributes["packages"],
                 **self.attributes["packages"],
@@ -106,7 +106,7 @@ class PyPIStepDecorator(StepDecorator):
         environment.set_local_root(LocalStorage.get_datastore_root_from_config(logger))
 
     def is_attribute_user_defined(self, name):
-        return name in self._attributes_with_user_values
+        return name in self._user_defined_attributes
 
 
 class PyPIFlowDecorator(FlowDecorator):
@@ -129,10 +129,9 @@ class PyPIFlowDecorator(FlowDecorator):
     defaults = {"packages": {}, "python": None, "disabled": None}
 
     def __init__(self, attributes=None, statically_defined=False):
-        self._attributes_with_user_values = (
-            set(attributes.keys()) if attributes is not None else set()
+        self._user_defined_attributes = (
+            attributes.copy() if attributes is not None else {}
         )
-
         super().__init__(attributes, statically_defined)
 
     def flow_init(
@@ -141,7 +140,6 @@ class PyPIFlowDecorator(FlowDecorator):
         from metaflow import decorators
 
         decorators._attach_decorators(flow, ["pypi"])
-        decorators._init(flow)
 
         # @pypi uses a conda environment to create a virtual environment.
         # The conda environment can be created through micromamba.

metaflow/plugins/timeout_decorator.py

@@ -37,8 +37,8 @@ class TimeoutDecorator(StepDecorator):
     name = "timeout"
     defaults = {"seconds": 0, "minutes": 0, "hours": 0}
 
-    def init(self):
-        super().init()
+    def __init__(self, *args, **kwargs):
+        super(TimeoutDecorator, self).__init__(*args, **kwargs)
         # Initialize secs in __init__ so other decorators could safely use this
         # value without worrying about decorator order.
         # Convert values in attributes to type:int since they can be type:str

metaflow/runner/click_api.py

@@ -9,7 +9,6 @@ if sys.version_info < (3, 7):
     )
 
 import datetime
-import functools
 import importlib
 import inspect
 import itertools
@@ -39,7 +38,6 @@ from metaflow.decorators import add_decorator_options
 from metaflow.exception import MetaflowException
 from metaflow.includefile import FilePathClass
 from metaflow.parameters import JSONTypeClass, flow_context
-from metaflow.user_configs.config_options import LocalFileInput
 
 # Define a recursive type alias for JSON
 JSON = Union[Dict[str, "JSON"], List["JSON"], str, int, float, bool, None]
@@ -57,7 +55,6 @@ click_to_python_types = {
     File: str,
     JSONTypeClass: JSON,
     FilePathClass: str,
-    LocalFileInput: str,
 }
 
 
@@ -127,37 +124,6 @@ def _method_sanity_check(
     return method_params
 
 
-def _lazy_load_command(
-    cli_collection: click.Group,
-    flow_parameters: Union[str, List[Parameter]],
-    _self,
-    name: str,
-):
-
-    # Context is not used in get_command so we can pass None. Since we pin click,
-    # this won't change from under us.
-
-    if isinstance(flow_parameters, str):
-        # Resolve flow_parameters -- for start, this is a function which we
-        # need to call to figure out the actual parameters (may be changed by configs)
-        flow_parameters = getattr(_self, flow_parameters)()
-    cmd_obj = cli_collection.get_command(None, name)
-    if cmd_obj:
-        if isinstance(cmd_obj, click.Group):
-            # TODO: possibly check for fake groups with cmd_obj.name in ["cli", "main"]
-            result = functools.partial(extract_group(cmd_obj, flow_parameters), _self)
-        elif isinstance(cmd_obj, click.Command):
-            result = functools.partial(extract_command(cmd_obj, flow_parameters), _self)
-        else:
-            raise RuntimeError(
-                "Cannot handle %s of type %s" % (cmd_obj.name, type(cmd_obj))
-            )
-        setattr(_self, name, result)
-        return result
-    else:
-        raise AttributeError()
-
-
 def get_annotation(param: Union[click.Argument, click.Option]):
     py_type = click_to_python_types[type(param.type)]
     if not param.required:
@@ -213,11 +179,9 @@ def extract_flow_class_from_file(flow_file: str) -> FlowSpec:
 
 
 class MetaflowAPI(object):
-    def __init__(self, parent=None, flow_cls=None, **kwargs):
+    def __init__(self, parent=None, **kwargs):
         self._parent = parent
         self._chain = [{self._API_NAME: kwargs}]
-        self._flow_cls = flow_cls
-        self._cached_computed_parameters = None
 
     @property
     def parent(self):
@@ -236,22 +200,23 @@ class MetaflowAPI(object):
     @classmethod
     def from_cli(cls, flow_file: str, cli_collection: Callable) -> Callable:
         flow_cls = extract_flow_class_from_file(flow_file)
-
+        flow_parameters = [p for _, p in flow_cls._get_parameters()]
         with flow_context(flow_cls) as _:
             add_decorator_options(cli_collection)
 
-        def getattr_wrapper(_self, name):
-            # Functools.partial do not automatically bind self (no __get__)
-            return _self._internal_getattr(_self, name)
-
-        class_dict = {
-            "__module__": "metaflow",
-            "_API_NAME": flow_file,
-            "_internal_getattr": functools.partial(
-                _lazy_load_command, cli_collection, "_compute_flow_parameters"
-            ),
-            "__getattr__": getattr_wrapper,
-        }
+        class_dict = {"__module__": "metaflow", "_API_NAME": flow_file}
+        command_groups = cli_collection.sources
+        for each_group in command_groups:
+            for _, cmd_obj in each_group.commands.items():
+                if isinstance(cmd_obj, click.Group):
+                    # TODO: possibly check for fake groups with cmd_obj.name in ["cli", "main"]
+                    class_dict[cmd_obj.name] = extract_group(cmd_obj, flow_parameters)
+                elif isinstance(cmd_obj, click.Command):
+                    class_dict[cmd_obj.name] = extract_command(cmd_obj, flow_parameters)
+                else:
+                    raise RuntimeError(
+                        "Cannot handle %s of type %s" % (cmd_obj.name, type(cmd_obj))
+                    )
 
         to_return = type(flow_file, (MetaflowAPI,), class_dict)
         to_return.__name__ = flow_file
@@ -272,11 +237,11 @@ class MetaflowAPI(object):
                 defaults,
                 **kwargs,
             )
-            return to_return(parent=None, flow_cls=flow_cls, **method_params)
+            return to_return(parent=None, **method_params)
 
         m = _method
-        m.__name__ = cli_collection.name
-        m.__doc__ = getattr(cli_collection, "help", None)
+        m.__name__ = cmd_obj.name
+        m.__doc__ = getattr(cmd_obj, "help", None)
         m.__signature__ = inspect.signature(_method).replace(
             parameters=params_sigs.values()
         )
@@ -322,25 +287,6 @@ class MetaflowAPI(object):
 
         return components
 
-    def _compute_flow_parameters(self):
-        if self._flow_cls is None or self._parent is not None:
-            raise RuntimeError(
-                "Computing flow-level parameters for a non start API. "
-                "Please report to the Metaflow team."
-            )
-        # TODO: We need to actually compute the new parameters (based on configs) which
-        # would involve processing the options at least partially. We will do this
-        # before GA but for now making it work for regular parameters
-        if self._cached_computed_parameters is not None:
-            return self._cached_computed_parameters
-        self._cached_computed_parameters = []
-        for _, param in self._flow_cls._get_parameters():
-            if param.IS_CONFIG_PARAMETER:
-                continue
-            param.init()
-            self._cached_computed_parameters.append(param)
-        return self._cached_computed_parameters
-
 
 def extract_all_params(cmd_obj: Union[click.Command, click.Group]):
     arg_params_sigs = OrderedDict()
@@ -405,7 +351,7 @@ def extract_group(cmd_obj: click.Group, flow_parameters: List[Parameter]) -> Cal
         method_params = _method_sanity_check(
             possible_arg_params, possible_opt_params, annotations, defaults, **kwargs
         )
-        return resulting_class(parent=_self, flow_cls=None, **method_params)
+        return resulting_class(parent=_self, **method_params)
 
     m = _method
     m.__name__ = cmd_obj.name

metaflow/runner/deployer.py

@@ -64,7 +64,7 @@ class Deployer(metaclass=DeployerMeta):
         The directory to run the subprocess in; if not specified, the current
         directory is used.
     file_read_timeout : int, default 3600
-        The timeout until which we try to read the deployer attribute file (in seconds).
+        The timeout until which we try to read the deployer attribute file.
     **kwargs : Any
         Additional arguments that you would pass to `python myflow.py` before
         the deployment command.

metaflow/runner/deployer_impl.py

@@ -37,7 +37,7 @@ class DeployerImpl(object):
         The directory to run the subprocess in; if not specified, the current
         directory is used.
     file_read_timeout : int, default 3600
-        The timeout until which we try to read the deployer attribute file (in seconds).
+        The timeout until which we try to read the deployer attribute file.
     **kwargs : Any
         Additional arguments that you would pass to `python myflow.py` before
         the deployment command.
@@ -144,7 +144,7 @@ class DeployerImpl(object):
             # Additional info is used to pass additional deployer specific information.
             # It is used in non-OSS deployers (extensions).
             self.additional_info = content.get("additional_info", {})
-            command_obj.sync_wait()
+
             if command_obj.process.returncode == 0:
                 return create_class(deployer=self)
 

metaflow/runner/metaflow_runner.py

@@ -221,7 +221,7 @@ class Runner(object):
         The directory to run the subprocess in; if not specified, the current
         directory is used.
     file_read_timeout : int, default 3600
-        The timeout until which we try to read the runner attribute file (in seconds).
+        The timeout until which we try to read the runner attribute file.
     **kwargs : Any
         Additional arguments that you would pass to `python myflow.py` before
         the `run` command.
@@ -272,9 +272,6 @@ class Runner(object):
 
     def __get_executing_run(self, attribute_file_fd, command_obj):
         content = handle_timeout(attribute_file_fd, command_obj, self.file_read_timeout)
-
-        command_obj.sync_wait()
-
         content = json.loads(content)
         pathspec = "%s/%s" % (content.get("flow_name"), content.get("run_id"))
 

metaflow/runner/nbdeploy.py

@@ -46,8 +46,6 @@ class NBDeployer(object):
     base_dir : str, optional, default None
         The directory to run the subprocess in; if not specified, the current
         working directory is used.
-    file_read_timeout : int, default 3600
-        The timeout until which we try to read the deployer attribute file (in seconds).
     **kwargs : Any
         Additional arguments that you would pass to `python myflow.py` i.e. options
         listed in `python myflow.py --help`

metaflow/runner/nbrun.py

@@ -44,7 +44,7 @@ class NBRunner(object):
         The directory to run the subprocess in; if not specified, the current
         working directory is used.
     file_read_timeout : int, default 3600
-        The timeout until which we try to read the runner attribute file (in seconds).
+        The timeout until which we try to read the runner attribute file.
     **kwargs : Any
         Additional arguments that you would pass to `python myflow.py` before
         the `run` command.

metaflow/runner/subprocess_manager.py

@@ -120,9 +120,6 @@ class SubprocessManager(object):
         """
         Run a command synchronously and return its process ID.
 
-        Note: in no case does this wait for the process to *finish*. Use sync_wait()
-        to wait for the command to finish.
-
         Parameters
         ----------
         command : List[str]
@@ -148,6 +145,7 @@ class SubprocessManager(object):
         command_obj = CommandManager(command, env, cwd)
         pid = command_obj.run(show_output=show_output)
         self.commands[pid] = command_obj
+        command_obj.sync_wait()
         return pid
 
     async def async_run_command(

metaflow/runner/utils.py

@@ -91,7 +91,7 @@ def read_from_fifo_when_ready(
     encoding : str, optional
         Encoding to use while reading the file, by default "utf-8".
     timeout : int, optional
-        Timeout for reading the file in seconds, by default 3600.
+        Timeout for reading the file in milliseconds, by default 3600.
 
     Returns
     -------
@@ -107,47 +107,30 @@ def read_from_fifo_when_ready(
         content to the FIFO.
     """
     content = bytearray()
+
     poll = select.poll()
     poll.register(fifo_fd, select.POLLIN)
-    max_timeout = 3  # Wait for 10 * 3 = 30 ms after last write
-    while True:
-        if timeout < 0:
-            raise TimeoutError("Timeout while waiting for the file content")
 
+    while True:
         poll_begin = time.time()
-        # We poll for a very short time to be also able to check if the file was closed
-        # If the file is closed, we assume that we only have one writer so if we have
-        # data, we break out. This is to work around issues in macos
-        events = poll.poll(min(10, timeout * 1000))
-        timeout -= time.time() - poll_begin
+        poll.poll(timeout)
+        timeout -= 1000 * (time.time() - poll_begin)
+
+        if timeout <= 0:
+            raise TimeoutError("Timeout while waiting for the file content")
 
         try:
-            data = os.read(fifo_fd, 8192)
-            if data:
+            data = os.read(fifo_fd, 128)
+            while data:
                 content += data
-            else:
-                if len(events):
-                    # We read an EOF -- consider the file done
-                    break
-                else:
-                    # We had no events (just a timeout) and the read didn't return
-                    # an exception so the file is still open; we continue waiting for data
-                    # Unfortunately, on MacOS, it seems that even *after* the file is
-                    # closed on the other end, we still don't get a BlockingIOError so
-                    # we hack our way and timeout if there is no write in 30ms which is
-                    # a relative eternity for file writes.
-                    if content:
-                        if max_timeout <= 0:
-                            break
-                        max_timeout -= 1
-                        continue
+                data = os.read(fifo_fd, 128)
+
+            # Read from a non-blocking closed FIFO returns an empty byte array
+            break
+
         except BlockingIOError:
-            has_blocking_error = True
-            if content:
-                # The file was closed
-                break
-            # else, if we have no content, we continue waiting for the file to be open
-            # and written to.
+            # FIFO is open but no data is available yet
+            continue
 
     if not content and check_process_exited(command_obj):
         raise CalledProcessError(command_obj.process.returncode, command_obj.command)
@@ -173,7 +156,7 @@ async def async_read_from_fifo_when_ready(
     encoding : str, optional
         Encoding to use while reading the file, by default "utf-8".
     timeout : int, optional
-        Timeout for reading the file in seconds, by default 3600.
+        Timeout for reading the file in milliseconds, by default 3600.
 
     Returns
     -------
@@ -223,7 +206,7 @@ def handle_timeout(
     command_obj : CommandManager
         Command manager object that encapsulates the running command details.
     file_read_timeout : int
-        Timeout for reading the file, in seconds
+        Timeout for reading the file.
 
     Returns
     -------
@@ -260,7 +243,7 @@ async def async_handle_timeout(
     command_obj : CommandManager
         Command manager object that encapsulates the running command details.
     file_read_timeout : int
-        Timeout for reading the file, in seconds
+        Timeout for reading the file.
 
     Returns
     -------