ob-metaflow 2.10.7.2__tar.gz → 2.10.7.4__tar.gz

{ob-metaflow-2.10.7.2/ob_metaflow.egg-info → ob-metaflow-2.10.7.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow
-Version: 2.10.7.2
+Version: 2.10.7.4
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co

{ob-metaflow-2.10.7.2 → ob-metaflow-2.10.7.4}/metaflow/plugins/argo/argo_workflows.py

@@ -1034,9 +1034,16 @@ class ArgoWorkflows(object):
             # Ideally, we would like these task ids to be the same as node name
             # (modulo retry suffix) on Argo Workflows but that doesn't seem feasible
             # right now.
+
             task_str = node.name + "-{{workflow.creationTimestamp}}"
+            # The input paths might be quite large for foreach joins,
+            # so we need to take care not to unnecessarily repeat these in the scripts due to a template max size limit with Argo.
+            # Note: We inline a Python script here because jq might not be present, and bundling this as an utility will not work
+            # as the metaflow module initializes only after this line.
+            # read input-paths directly from ARGO_TEMPLATE environment variable.
+            input_paths_expr = """export INPUT_PATHS=$(python -c \\"import sys, os, json; template = json.loads(os.environ['ARGO_TEMPLATE']); input_paths = next((param['value'] for param in template.get('inputs', {'parameters': []})['parameters'] if param['name'] == 'input-paths'), None); sys.stdout.write(input_paths if input_paths else '')\\")"""
             if node.name != "start":
-                task_str += "-{{inputs.parameters.input-paths}}"
+                task_str += "-$(echo $INPUT_PATHS)"
             if any(self.graph[n].type == "foreach" for n in node.in_funcs):
                 task_str += "-{{inputs.parameters.split-index}}"
             # Generated task_ids need to be non-numeric - see register_task_id in
@@ -1087,6 +1094,7 @@ class ArgoWorkflows(object):
                     # env var.
                     '${METAFLOW_INIT_SCRIPT:+eval \\"${METAFLOW_INIT_SCRIPT}\\"}',
                     "mkdir -p $PWD/.logs",
+                    input_paths_expr,
                     task_id_expr,
                     mflog_expr,
                 ]
@@ -1098,7 +1106,7 @@ class ArgoWorkflows(object):
                 node.name, self.flow_datastore.TYPE
             )
 
-            input_paths = "{{inputs.parameters.input-paths}}"
+            input_paths = "$(echo $INPUT_PATHS)"
 
             top_opts_dict = {
                 "with": [
@@ -2835,4 +2843,4 @@ class LifecycleHook(object):
         return self.payload
 
     def __str__(self):
-        return json.dumps(self.payload, indent=4)
+        return json.dumps(self.payload, indent=4)

{ob-metaflow-2.10.7.2 → ob-metaflow-2.10.7.4}/metaflow/plugins/argo/process_input_paths.py

@@ -11,9 +11,9 @@ def process_input_paths(input_paths):
     flow, run_id, task_ids = input_paths.split("/")
     task_ids = re.sub("[\[\]{}]", "", task_ids)
     task_ids = task_ids.split(",")
-    tasks = [t.split(":")[1] for t in task_ids]
+    tasks = [t.split(":")[1].strip('"') for t in task_ids]
     return "{}/{}/:{}".format(flow, run_id, ",".join(tasks))
 
 
 if __name__ == "__main__":
-    print(process_input_paths(sys.argv[1]))
+    print(process_input_paths(sys.argv[1]))

{ob-metaflow-2.10.7.2 → ob-metaflow-2.10.7.4}/metaflow/tracing/span_exporter.py

@@ -32,13 +32,13 @@ def set_otel_exporter():
         span_exporter = OTLPSpanExporter(
             endpoint=OTEL_ENDPOINT,
             headers={"x-api-key": SERVICE_AUTH_KEY},
-            timeout=1,
+            timeout=15,
         )
     elif SERVICE_HEADERS:
         span_exporter = OTLPSpanExporter(
             endpoint=OTEL_ENDPOINT,
             headers=SERVICE_HEADERS,
-            timeout=1,
+            timeout=15,
         )
     else:
         print("WARNING: no auth settings for Opentelemetry", file=sys.stderr)

ob-metaflow-2.10.7.4/metaflow/version.py

@@ -0,0 +1 @@
+metaflow_version = "2.10.7.4"

{ob-metaflow-2.10.7.2 → ob-metaflow-2.10.7.4/ob_metaflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow
-Version: 2.10.7.2
+Version: 2.10.7.4
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co

ob-metaflow-2.10.7.2/metaflow/version.py

@@ -1 +0,0 @@
-metaflow_version = "2.10.7.2"