ob-metaflow 2.10.7.2__py2.py3-none-any.whl → 2.10.7.4__py2.py3-none-any.whl

metaflow/plugins/argo/argo_workflows.py

@@ -1034,9 +1034,16 @@ class ArgoWorkflows(object):
             # Ideally, we would like these task ids to be the same as node name
             # (modulo retry suffix) on Argo Workflows but that doesn't seem feasible
             # right now.
+
             task_str = node.name + "-{{workflow.creationTimestamp}}"
+            # The input paths might be quite large for foreach joins,
+            # so we need to take care not to unnecessarily repeat these in the scripts due to a template max size limit with Argo.
+            # Note: We inline a Python script here because jq might not be present, and bundling this as an utility will not work
+            # as the metaflow module initializes only after this line.
+            # read input-paths directly from ARGO_TEMPLATE environment variable.
+            input_paths_expr = """export INPUT_PATHS=$(python -c \\"import sys, os, json; template = json.loads(os.environ['ARGO_TEMPLATE']); input_paths = next((param['value'] for param in template.get('inputs', {'parameters': []})['parameters'] if param['name'] == 'input-paths'), None); sys.stdout.write(input_paths if input_paths else '')\\")"""
             if node.name != "start":
-                task_str += "-{{inputs.parameters.input-paths}}"
+                task_str += "-$(echo $INPUT_PATHS)"
             if any(self.graph[n].type == "foreach" for n in node.in_funcs):
                 task_str += "-{{inputs.parameters.split-index}}"
             # Generated task_ids need to be non-numeric - see register_task_id in
@@ -1087,6 +1094,7 @@ class ArgoWorkflows(object):
                     # env var.
                     '${METAFLOW_INIT_SCRIPT:+eval \\"${METAFLOW_INIT_SCRIPT}\\"}',
                     "mkdir -p $PWD/.logs",
+                    input_paths_expr,
                     task_id_expr,
                     mflog_expr,
                 ]
@@ -1098,7 +1106,7 @@ class ArgoWorkflows(object):
                 node.name, self.flow_datastore.TYPE
             )
 
-            input_paths = "{{inputs.parameters.input-paths}}"
+            input_paths = "$(echo $INPUT_PATHS)"
 
             top_opts_dict = {
                 "with": [
@@ -2835,4 +2843,4 @@ class LifecycleHook(object):
         return self.payload
 
     def __str__(self):
-        return json.dumps(self.payload, indent=4)
+        return json.dumps(self.payload, indent=4)

metaflow/plugins/argo/process_input_paths.py

@@ -11,9 +11,9 @@ def process_input_paths(input_paths):
     flow, run_id, task_ids = input_paths.split("/")
     task_ids = re.sub("[\[\]{}]", "", task_ids)
     task_ids = task_ids.split(",")
-    tasks = [t.split(":")[1] for t in task_ids]
+    tasks = [t.split(":")[1].strip('"') for t in task_ids]
     return "{}/{}/:{}".format(flow, run_id, ",".join(tasks))
 
 
 if __name__ == "__main__":
-    print(process_input_paths(sys.argv[1]))
+    print(process_input_paths(sys.argv[1]))

metaflow/tracing/span_exporter.py

@@ -32,13 +32,13 @@ def set_otel_exporter():
         span_exporter = OTLPSpanExporter(
             endpoint=OTEL_ENDPOINT,
             headers={"x-api-key": SERVICE_AUTH_KEY},
-            timeout=1,
+            timeout=15,
         )
     elif SERVICE_HEADERS:
         span_exporter = OTLPSpanExporter(
             endpoint=OTEL_ENDPOINT,
             headers=SERVICE_HEADERS,
-            timeout=1,
+            timeout=15,
         )
     else:
         print("WARNING: no auth settings for Opentelemetry", file=sys.stderr)

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.10.7.2"
+metaflow_version = "2.10.7.4"

{ob_metaflow-2.10.7.2.dist-info → ob_metaflow-2.10.7.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow
-Version: 2.10.7.2
+Version: 2.10.7.4
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co

{ob_metaflow-2.10.7.2.dist-info → ob_metaflow-2.10.7.4.dist-info}/RECORD

@@ -33,7 +33,7 @@ metaflow/task.py,sha256=summMHj2oQ37_Izp3c78FzS67jJkoKjtgLzYJUVXcs8,25278
 metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
 metaflow/util.py,sha256=jbMJ17rK-dFTBCjimWqxkfcr3v__bHa3tZtX0g8iS2c,13257
 metaflow/vendor.py,sha256=LZgXrh7ZSDmD32D1T5jj3OKKpXIqqxKzdMAOc5V0SD4,5162
-metaflow/version.py,sha256=ZjwWEP7zb6kYoyuNI_dsyE-o49dafPdFPeWFGOsabpU,30
+metaflow/version.py,sha256=NwHUaPr0oX_XJ9sy4vMuKheohP5q3dzabVLLJjy41r4,30
 metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
 metaflow/_vendor/click/__init__.py,sha256=FkyGDQ-cbiQxP_lxgUspyFYS48f2S_pTcfKPz-d_RMo,2463
 metaflow/_vendor/click/_bashcomplete.py,sha256=9J98IHQYmCAr2Jup6TDshUr5FJEen-AoQCZR0K5nKxQ,12309
@@ -145,10 +145,10 @@ metaflow/plugins/airflow/sensors/s3_sensor.py,sha256=zym4mUm_f_gBsvHHVqGtX_OOxRj
 metaflow/plugins/argo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/plugins/argo/argo_client.py,sha256=MKKhMCbWOPzf6z5zQQiyDRHHkAXcO7ipboDZDqAAvOk,15849
 metaflow/plugins/argo/argo_events.py,sha256=TIEOHrvUQ079YzzpzpFFtZjeU2x7hoofyZ6ytCorm2c,5911
-metaflow/plugins/argo/argo_workflows.py,sha256=5-f7EJtRlDWjOkmWj1zZqYM5oXXca-Xtc7rEV98_coQ,118896
+metaflow/plugins/argo/argo_workflows.py,sha256=du8R0vI2R0_Z2c-MOAdLldOCkoHl2c-G2mDbARKysV4,119741
 metaflow/plugins/argo/argo_workflows_cli.py,sha256=Abp3xItIku-sT-q84WhNoPUCwlNgosrkZLLObJwA0io,33258
 metaflow/plugins/argo/argo_workflows_decorator.py,sha256=CfKVoHCOsCCQMghhPE30xw15gacwp3hR23HCo9ZZFVg,6580
-metaflow/plugins/argo/process_input_paths.py,sha256=4SiUoxbnTX4rCt0RSLcxG5jysbyd8oU-5JT0UOgy-vk,555
+metaflow/plugins/argo/process_input_paths.py,sha256=LjUSP8PVU-DRGEPxjas99nzyAO-fI82Bxxbr_QETE88,565
 metaflow/plugins/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/plugins/aws/aws_client.py,sha256=mO8UD6pxFaOnxDb3hTP3HB7Gqb_ZxoR-76LT683WHvI,4036
 metaflow/plugins/aws/aws_utils.py,sha256=YtwrUwSfl0BBrQpQF_DrdroxTYyCJwdsQKzmOWcb01c,7199
@@ -267,7 +267,7 @@ metaflow/sidecar/sidecar_subprocess.py,sha256=8ZyIHA7p4Tx6Vge4RjNCxdEa6SpdYtPajG
 metaflow/sidecar/sidecar_worker.py,sha256=4DfpxtnuphngOnIehKjNR_Knhu1hY7DYBcHl4Svpe3Y,2050
 metaflow/tracing/__init__.py,sha256=xYTOT5BS5jbwhjk6hskxqNSU9st2LYtfeLN2Hknm3EI,1551
 metaflow/tracing/propagator.py,sha256=AdPeAqoeRauH82pTw01hLFNPRAzm29nlwM7C2iqKFFk,2502
-metaflow/tracing/span_exporter.py,sha256=kbi92INM8yKPqwLEVbWy7OcdsVJoh5nCbwAK9VyPx_w,1520
+metaflow/tracing/span_exporter.py,sha256=Y7eYyaxcWFO5oeORpJ-Q9o0p3qe5qpjkutwPJ0LyLOI,1522
 metaflow/tracing/tracing_modules.py,sha256=SK2ia5q1Vm8LTdFZeaax0pu3al9OwgcLVow5IE6KSaY,4680
 metaflow/tutorials/00-helloworld/README.md,sha256=ML9k-2mRZOAFV3RDzwAWSIjnWzG03-3n9ov59UAvN5k,386
 metaflow/tutorials/00-helloworld/helloworld.py,sha256=ji0v7TFjVVR0VTs-shQOswh1Zd0YP5rZxud_LEmsjHU,857
@@ -292,9 +292,9 @@ metaflow/tutorials/07-worldview/README.md,sha256=5vQTrFqulJ7rWN6r20dhot9lI2sVj9W
 metaflow/tutorials/07-worldview/worldview.ipynb,sha256=ztPZPI9BXxvW1QdS2Tfe7LBuVzvFvv0AToDnsDJhLdE,2237
 metaflow/tutorials/08-autopilot/README.md,sha256=GnePFp_q76jPs991lMUqfIIh5zSorIeWznyiUxzeUVE,1039
 metaflow/tutorials/08-autopilot/autopilot.ipynb,sha256=DQoJlILV7Mq9vfPBGW-QV_kNhWPjS5n6SJLqePjFYLY,3191
-ob_metaflow-2.10.7.2.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
-ob_metaflow-2.10.7.2.dist-info/METADATA,sha256=UDz6FFz9S8aWwQ7ju2fstbz0eHvH786Ik_QpDLMNioc,5061
-ob_metaflow-2.10.7.2.dist-info/WHEEL,sha256=-G_t0oGuE7UD0DrSpVZnq1hHMBV9DD2XkS5v7XpmTnk,110
-ob_metaflow-2.10.7.2.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
-ob_metaflow-2.10.7.2.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
-ob_metaflow-2.10.7.2.dist-info/RECORD,,
+ob_metaflow-2.10.7.4.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
+ob_metaflow-2.10.7.4.dist-info/METADATA,sha256=d_cqDlorVjUzb41ZJmc2auwSNw8WSM9B-ZyhBYqQIhM,5061
+ob_metaflow-2.10.7.4.dist-info/WHEEL,sha256=-G_t0oGuE7UD0DrSpVZnq1hHMBV9DD2XkS5v7XpmTnk,110
+ob_metaflow-2.10.7.4.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
+ob_metaflow-2.10.7.4.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
+ob_metaflow-2.10.7.4.dist-info/RECORD,,