ob-metaflow 2.12.27.1__py2.py3-none-any.whl → 2.12.30.1__py2.py3-none-any.whl

metaflow/datastore/task_datastore.py

@@ -10,7 +10,7 @@ from types import MethodType, FunctionType
 
 from .. import metaflow_config
 from ..exception import MetaflowInternalError
-from ..metadata import DataArtifact, MetaDatum
+from ..metadata_provider import DataArtifact, MetaDatum
 from ..parameters import Parameter
 from ..util import Path, is_stringish, to_fileobj
 

metaflow/extension_support/plugins.py

@@ -178,6 +178,7 @@ _plugin_categories = {
     "environment": lambda x: x.TYPE,
     "metadata_provider": lambda x: x.TYPE,
     "datastore": lambda x: x.TYPE,
+    "dataclient": lambda x: x.TYPE,
     "secrets_provider": lambda x: x.TYPE,
     "gcp_client_provider": lambda x: x.name,
     "deployer_impl_provider": lambda x: x.TYPE,

metaflow/flowspec.py

@@ -64,7 +64,7 @@ class ParallelUBF(UnboundedForeachInput):
         return item or 0  # item is None for the control task, but it is also split 0
 
 
-class _FlowSpecMeta(type):
+class FlowSpecMeta(type):
     def __new__(cls, name, bases, dct):
         f = super().__new__(cls, name, bases, dct)
         # This makes sure to give _flow_decorators to each
@@ -75,7 +75,7 @@ class _FlowSpecMeta(type):
         return f
 
 
-class FlowSpec(metaclass=_FlowSpecMeta):
+class FlowSpec(metaclass=FlowSpecMeta):
     """
     Main class from which all Flows should inherit.
 

metaflow/includefile.py

@@ -1,6 +1,7 @@
 from collections import namedtuple
 import gzip
 
+import importlib
 import io
 import json
 import os
@@ -17,6 +18,8 @@ from .parameters import (
     Parameter,
     ParameterContext,
 )
+
+from .plugins import DATACLIENTS
 from .util import get_username
 
 import functools
@@ -47,16 +50,7 @@ _DelayedExecContext = namedtuple(
 
 
 # From here on out, this is the IncludeFile implementation.
-from metaflow.plugins.datatools import Local, S3
-from metaflow.plugins.azure.includefile_support import Azure
-from metaflow.plugins.gcp.includefile_support import GS
-
-DATACLIENTS = {
-    "local": Local,
-    "s3": S3,
-    "azure": Azure,
-    "gs": GS,
-}
+_dict_dataclients = {d.TYPE: d for d in DATACLIENTS}
 
 
 class IncludedFile(object):
@@ -167,7 +161,7 @@ class FilePathClass(click.ParamType):
                 "IncludeFile using a direct reference to a file in cloud storage is no "
                 "longer supported. Contact the Metaflow team if you need this supported"
             )
-            # if DATACLIENTS.get(path[:prefix_pos]) is None:
+            # if _dict_dataclients.get(path[:prefix_pos]) is None:
             #     self.fail(
             #         "IncludeFile: no handler for external file of type '%s' "
             #         "(given path is '%s')" % (path[:prefix_pos], path)
@@ -187,7 +181,7 @@ class FilePathClass(click.ParamType):
                     pass
             except OSError:
                 self.fail("IncludeFile: could not open file '%s' for reading" % path)
-            handler = DATACLIENTS.get(ctx.ds_type)
+            handler = _dict_dataclients.get(ctx.ds_type)
             if handler is None:
                 self.fail(
                     "IncludeFile: no data-client for datastore of type '%s'"
@@ -213,7 +207,7 @@ class FilePathClass(click.ParamType):
                         ctx.path,
                         ctx.is_text,
                         ctx.encoding,
-                        DATACLIENTS[ctx.handler_type],
+                        _dict_dataclients[ctx.handler_type],
                         ctx.echo,
                     )
                 )
@@ -425,7 +419,7 @@ class UploaderV1:
         if prefix_pos < 0:
             raise MetaflowException("Malformed URL: '%s'" % url)
         prefix = url[:prefix_pos]
-        handler = DATACLIENTS.get(prefix)
+        handler = _dict_dataclients.get(prefix)
         if handler is None:
             raise MetaflowException("Could not find data client for '%s'" % prefix)
         return handler

metaflow/metaflow_config.py

@@ -43,6 +43,10 @@ DEFAULT_GCP_CLIENT_PROVIDER = from_conf("DEFAULT_GCP_CLIENT_PROVIDER", "gcp-defa
 DEFAULT_SECRETS_BACKEND_TYPE = from_conf("DEFAULT_SECRETS_BACKEND_TYPE")
 DEFAULT_SECRETS_ROLE = from_conf("DEFAULT_SECRETS_ROLE")
 
+DEFAULT_FROM_DEPLOYMENT_IMPL = from_conf(
+    "DEFAULT_FROM_DEPLOYMENT_IMPL", "argo-workflows"
+)
+
 ###
 # User configuration
 ###
@@ -411,13 +415,6 @@ AIRFLOW_KUBERNETES_KUBECONFIG_CONTEXT = from_conf(
 )
 
 
-###
-# From Deployment configuration
-###
-# From Deployment Configuration for DeployedFlow
-FROM_DEPLOYMENT_IMPL = from_conf("FROM_DEPLOYMENT_IMPL", "argo-workflows")
-
-
 ###
 # Conda configuration
 ###

metaflow/metaflow_current.py

@@ -30,7 +30,7 @@ class Current(object):
             raise ex
 
         self.__class__.graph = property(
-            fget=lambda _: _raise(RuntimeError("Graph is not available"))
+            fget=lambda self: _raise(RuntimeError("Graph is not available"))
         )
 
     def _set_env(

metaflow/parameters.py

@@ -438,3 +438,6 @@ def add_custom_parameters(deploy_mode=False):
         return cmd
 
     return wrapper
+
+
+JSONType = JSONTypeClass()

metaflow/plugins/__init__.py

@@ -73,8 +73,8 @@ ENVIRONMENTS_DESC = [
 
 # Add metadata providers here
 METADATA_PROVIDERS_DESC = [
-    ("service", ".metadata.service.ServiceMetadataProvider"),
-    ("local", ".metadata.local.LocalMetadataProvider"),
+    ("service", ".metadata_providers.service.ServiceMetadataProvider"),
+    ("local", ".metadata_providers.local.LocalMetadataProvider"),
 ]
 
 # Add datastore here
@@ -85,13 +85,21 @@ DATASTORES_DESC = [
     ("gs", ".datastores.gs_storage.GSStorage"),
 ]
 
+# Dataclients are used for IncludeFile
+DATACLIENTS_DESC = [
+    ("local", ".datatools.Local"),
+    ("s3", ".datatools.S3"),
+    ("azure", ".azure.includefile_support.Azure"),
+    ("gs", ".gcp.includefile_support.GS"),
+]
+
 # Add non monitoring/logging sidecars here
 SIDECARS_DESC = [
     (
         "save_logs_periodically",
         "..mflog.save_logs_periodically.SaveLogsPeriodicallySidecar",
     ),
-    ("heartbeat", "metaflow.metadata.heartbeat.MetadataHeartBeat"),
+    ("heartbeat", "metaflow.metadata_provider.heartbeat.MetadataHeartBeat"),
 ]
 
 # Add logging sidecars here
@@ -156,16 +164,12 @@ def get_plugin_cli():
     return resolve_plugins("cli")
 
 
-# just a normal dictionary, not a plugin since there is no class..
-FROM_DEPLOYMENT_PROVIDERS = {
-    "argo-workflows": "metaflow.plugins.argo.argo_workflows_deployer",
-}
-
 STEP_DECORATORS = resolve_plugins("step_decorator")
 FLOW_DECORATORS = resolve_plugins("flow_decorator")
 ENVIRONMENTS = resolve_plugins("environment")
 METADATA_PROVIDERS = resolve_plugins("metadata_provider")
 DATASTORES = resolve_plugins("datastore")
+DATACLIENTS = resolve_plugins("dataclient")
 SIDECARS = resolve_plugins("sidecar")
 LOGGING_SIDECARS = resolve_plugins("logging_sidecar")
 MONITOR_SIDECARS = resolve_plugins("monitor_sidecar")

metaflow/plugins/airflow/airflow_cli.py

@@ -389,6 +389,11 @@ def _validate_workflow(flow, graph, flow_datastore, metadata, workflow_timeout):
                 "Step *%s* is marked for execution on AWS Batch with Airflow which isn't currently supported."
                 % node.name
             )
+        if any([d.name == "slurm" for d in node.decorators]):
+            raise NotSupportedException(
+                "Step *%s* is marked for execution on Slurm with Airflow which isn't currently supported."
+                % node.name
+            )
     SUPPORTED_DATASTORES = ("azure", "s3", "gs")
     if flow_datastore.TYPE not in SUPPORTED_DATASTORES:
         raise AirflowException(

metaflow/plugins/airflow/airflow_decorator.py

@@ -1,7 +1,7 @@
 import json
 import os
 from metaflow.decorators import StepDecorator
-from metaflow.metadata import MetaDatum
+from metaflow.metadata_provider import MetaDatum
 
 from .airflow_utils import (
     TASK_ID_XCOM_KEY,

metaflow/plugins/argo/argo_workflows_decorator.py

@@ -6,7 +6,7 @@ import time
 from metaflow import current
 from metaflow.decorators import StepDecorator
 from metaflow.events import Trigger
-from metaflow.metadata import MetaDatum
+from metaflow.metadata_provider import MetaDatum
 from metaflow.metaflow_config import ARGO_EVENTS_WEBHOOK_URL
 from metaflow.graph import DAGNode, FlowGraph
 from metaflow.flowspec import FlowSpec