runnable 0.36.1__py3-none-any.whl → 0.37.0__py3-none-any.whl

extensions/catalog/any_path.py

@@ -95,7 +95,10 @@ class AnyPathCatalog(BaseCatalog):
         return data_catalogs
 
     def put(
-        self, name: str, allow_file_not_found_exc: bool = False
+        self,
+        name: str,
+        allow_file_not_found_exc: bool = False,
+        store_copy: bool = True,
     ) -> List[DataCatalog]:
         """
         Put the files matching the glob pattern into the catalog.
@@ -154,7 +157,15 @@ class AnyPathCatalog(BaseCatalog):
             data_catalogs.append(data_catalog)
 
             # TODO: Think about syncing only if the file is changed
-            self.upload_to_catalog(file)
+            if store_copy:
+                logger.debug(
+                    f"Copying file {file} to the catalog location for run_id: {run_id}"
+                )
+                self.upload_to_catalog(file)
+            else:
+                logger.debug(
+                    f"Not copying file {file} to the catalog location for run_id: {run_id}"
+                )
 
         if not data_catalogs and not allow_file_not_found_exc:
             raise Exception(f"Did not find any files matching {name} in {copy_from}")

extensions/job_executor/__init__.py

@@ -29,6 +29,7 @@ class GenericJobExecutor(BaseJobExecutor):
     @property
     def _context(self):
         assert context.run_context
+        assert isinstance(context.run_context, context.JobContext)
         return context.run_context
 
     def _get_parameters(self) -> Dict[str, JsonParameter]:
@@ -147,7 +148,9 @@ class GenericJobExecutor(BaseJobExecutor):
         data_catalogs = []
         for name_pattern in catalog_settings:
             data_catalog = self._context.catalog.put(
-                name=name_pattern, allow_file_not_found_exc=allow_file_not_found_exc
+                name=name_pattern,
+                allow_file_not_found_exc=allow_file_not_found_exc,
+                store_copy=self._context.catalog_store_copy,
             )
 
             logger.debug(f"Added data catalog: {data_catalog} to job log")

extensions/pipeline_executor/__init__.py

@@ -160,7 +160,9 @@ class GenericPipelineExecutor(BasePipelineExecutor):
 
             elif stage == "put":
                 data_catalog = self._context.catalog.put(
-                    name=name_pattern, allow_file_not_found_exc=allow_file_no_found_exc
+                    name=name_pattern,
+                    allow_file_not_found_exc=allow_file_no_found_exc,
+                    store_copy=node_catalog_settings.get("store_copy", True),
                 )
             else:
                 raise Exception(f"Stage {stage} not supported")

runnable/catalog.py

@@ -57,7 +57,7 @@ class BaseCatalog(ABC, BaseModel):
 
     @abstractmethod
     def put(
-        self, name: str, allow_file_not_found_exc: bool = False
+        self, name: str, allow_file_not_found_exc: bool = False, store_copy: bool = True
     ) -> List[DataCatalog]:
         """
         Put the file by 'name' from the 'compute_data_folder' in the catalog for the run_id.
@@ -120,7 +120,10 @@ class DoNothingCatalog(BaseCatalog):
         return []
 
     def put(
-        self, name: str, allow_file_not_found_exc: bool = False
+        self,
+        name: str,
+        allow_file_not_found_exc: bool = False,
+        store_copy: bool = True,
     ) -> List[DataCatalog]:
         """
         Does nothing

runnable/context.py

@@ -475,6 +475,7 @@ class JobContext(RunnableContext):
         default=None,
         description="Catalog settings to be used for the job.",
     )
+    catalog_store_copy: bool = Field(default=True, alias="catalog_store_copy")
 
     @computed_field  # type: ignore
     @cached_property

runnable/nodes.py

@@ -411,11 +411,13 @@ class TraversalNode(BaseNode):
         return self.overrides.get(executor_type) or ""
 
 
+# Unfortunately, this is defined in 2 places. Look in SDK
 class CatalogStructure(BaseModel):
     model_config = ConfigDict(extra="forbid")  # Need to forbid
 
     get: List[str] = Field(default_factory=list)
     put: List[str] = Field(default_factory=list)
+    store_copy: bool = Field(default=True, alias="store_copy")
 
 
 class ExecutableNode(TraversalNode):

runnable/sdk.py

@@ -60,6 +60,7 @@ class Catalog(BaseModel):
     Attributes:
         get (List[str]): List of glob patterns to get from central catalog to the compute data folder.
         put (List[str]): List of glob patterns to put into central catalog from the compute data folder.
+        store_copy (bool): Whether to store a copy of the data in the central catalog.
 
     Examples:
         >>> from runnable import Catalog
@@ -74,6 +75,7 @@ class Catalog(BaseModel):
     # compute_data_folder: str = Field(default="", alias="compute_data_folder")
     get: List[str] = Field(default_factory=list, alias="get")
     put: List[str] = Field(default_factory=list, alias="put")
+    store_copy: bool = Field(default=True, alias="store_copy")
 
 
 class BaseTraversal(ABC, BaseModel):
@@ -845,6 +847,11 @@ class BaseJob(BaseModel):
             return []
         return self.catalog.put
 
+    def return_bool_catalog_store_copy(self) -> bool:
+        if self.catalog is None:
+            return True
+        return self.catalog.store_copy
+
     def _is_called_for_definition(self) -> bool:
         """
         If the run context is set, we are coming in only to get the pipeline definition.
@@ -888,6 +895,7 @@ class BaseJob(BaseModel):
         }
 
         run_context = context.JobContext.model_validate(configurations)
+        run_context.catalog_store_copy = self.return_bool_catalog_store_copy()
 
         assert isinstance(run_context.job_executor, BaseJobExecutor)
 

{runnable-0.36.1.dist-info → runnable-0.37.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: runnable
-Version: 0.36.1
+Version: 0.37.0
 Summary: Add your description here
 Author-email: "Vammi, Vijay" <vijay.vammi@astrazeneca.com>
 License-File: LICENSE

{runnable-0.36.1.dist-info → runnable-0.37.0.dist-info}/RECORD

@@ -1,13 +1,13 @@
 extensions/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 extensions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 extensions/catalog/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-extensions/catalog/any_path.py,sha256=atB5gWPRX6ptW6zwYeCVb_fh0qhs7WAFO9HIsnMZl98,7350
+extensions/catalog/any_path.py,sha256=NUZjXwOE-osR9RIpi59UC_16GZPCbkjshJL6X6hPmYw,7725
 extensions/catalog/file_system.py,sha256=T_qFPFfrmykoAMc1rjNi_DBb437me8WPRcFglwAK744,1767
 extensions/catalog/minio.py,sha256=R3GvfCxN1GTcs4bQIAWh79_GHDTVd14gnpKlzwFeKUI,2363
 extensions/catalog/pyproject.toml,sha256=lLNxY6v04c8I5QK_zKw_E6sJTArSJRA_V-79ktaA3Hk,279
 extensions/catalog/s3.py,sha256=Sw5t8_kVRprn3uGGJCiHn7M9zw1CLaCOFj6YErtfG0o,287
 extensions/job_executor/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-extensions/job_executor/__init__.py,sha256=5_pKw7GJbwmXL-QPdZwtfRKVHuKiPqLheK3rGU5G7-M,5813
+extensions/job_executor/__init__.py,sha256=9n2V4cLgwNuci0_62BvJTGA3Gf9h-aMUlAO-mKIOqYA,5958
 extensions/job_executor/emulate.py,sha256=d9KI4If0DeRkKv9iCSKyZCziK_3ARsJvs5RUFo9yigc,3220
 extensions/job_executor/k8s.py,sha256=W14u1xwJDtb-gyGdNl62IwWIs49AUO2oTAAxQCm6YT0,16456
 extensions/job_executor/k8s_job_spec.yaml,sha256=7aFpxHdO_p6Hkc3YxusUOuAQTD1Myu0yTPX9DrhxbOg,1158
@@ -25,7 +25,7 @@ extensions/nodes/stub.py,sha256=o9DjBekNa9O4b0VQOiNOA9eNjJ3C2a9Sn9d2fX7KaWg,2715
 extensions/nodes/success.py,sha256=yT4WkUI-1YN6qy4Ji6zSoZFXI9jOl6Ond4WxZRq-33k,2179
 extensions/nodes/task.py,sha256=m5_s5u7lQDoTmeO6CG94btrG5If9AKm2-RvOzvm0fDc,3147
 extensions/pipeline_executor/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-extensions/pipeline_executor/__init__.py,sha256=u1wLhQfDB1WotCoLIg1mwGlJTupfV312xACwjUxsbJw,24521
+extensions/pipeline_executor/__init__.py,sha256=JqJFGgLOUaDINUi4_xC3zoDSZxDrUYLNuHQVzr8-ByE,24620
 extensions/pipeline_executor/argo.py,sha256=5nEItAv-gNVi2sylZA4spKLWfcKuJ3zyQx3RekoLb28,38240
 extensions/pipeline_executor/emulate.py,sha256=ziaIzEpYgVOm2PPCY8bnRdoevMwS8OY9aEUXpwCFQR0,3547
 extensions/pipeline_executor/local.py,sha256=_8dtj88mI3xeZT4UPwtHNtbyi_VVzz_lpIxvC0UZS-g,1887
@@ -48,9 +48,9 @@ extensions/secrets/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,
 extensions/secrets/dotenv.py,sha256=nADHXI6KJ_LUYOIe5EbtYH-21OBebSNVr0Pjb1GlZ7w,1573
 extensions/secrets/pyproject.toml,sha256=mLJNImNcBlbLKHh-0ugVWT9V83R4RibyyYDtBCSqVF4,282
 runnable/__init__.py,sha256=MN9x2jmQb2eOr-rap1DXLzNSC926U-aad_YwENzG52w,509
-runnable/catalog.py,sha256=A03iMLW8S6_AZjucBoFhl7fPKx-khTLQeICVAUjd9WU,3767
+runnable/catalog.py,sha256=6l0tT0jwHi40aE6fhQMgYtYe_-2As-bRKztAKiFvy3o,3842
 runnable/cli.py,sha256=CziCKBoL-dklSEo_x-YO1AysrG2eaf2LMQZbcNTeCbM,7283
-runnable/context.py,sha256=yEJodj6hbBAcvpQdoAUAwoYXsqiJTQtxmGTJWIaexHs,17515
+runnable/context.py,sha256=mLpq5rtMsPawjnaN9Woq7HWZ1FAppeudZtYMT5vf6Fo,17594
 runnable/datastore.py,sha256=2pYg4i1JRMzw_CUUIsPOWt7wYPiGBamfo-CPVAkEH54,32375
 runnable/defaults.py,sha256=4UYuShnjEyWP529UlFnubvkBpOcczKIdE4jEOhPBwl4,3076
 runnable/entrypoints.py,sha256=46prgr3_FYtBMlRbUXIDSpgZUBgaxcdJAekXhgEIj7M,6578
@@ -58,15 +58,15 @@ runnable/exceptions.py,sha256=t5tSlYqe_EjU5liXu32yLLh_yrnXeFL93BuXfmQzV98,3268
 runnable/executor.py,sha256=CwzHkeGVpocACZLzfFS94TzKeiaPLv4NtXtvT3eoocY,15222
 runnable/graph.py,sha256=poQz5zcvq89ju_u5sYlunQLPbHnXTaUmjcvstPwvT4U,16536
 runnable/names.py,sha256=A9ldUyULXuWjJ1MoXihHqlg-xeTVX-oWYTO5Ah0trmo,8128
-runnable/nodes.py,sha256=JXB7TLA5zrFFk1cJXLrSBfunp4aVRFw_Y5uIjIGOpHg,16805
+runnable/nodes.py,sha256=JHBxJib7SSQXY51bLHBXUvb0DlNSLNvyqz3JNEDLt8c,16926
 runnable/parameters.py,sha256=zEehAliVvCOLOnNZ4ExJvSDJM_2PWY0URZ0bmZUgCQA,5289
 runnable/pickler.py,sha256=ydJ_eti_U1F4l-YacFp7BWm6g5vTn04UXye25S1HVok,2684
-runnable/sdk.py,sha256=C7U_E9viXjTupT13hXM4lPNHGxsRqqvhpUUitMuq_CU,32207
+runnable/sdk.py,sha256=blLBWzXV2x7jxKQXWpjmeJ9k22jt5CKBQBqQpnt4agk,32587
 runnable/secrets.py,sha256=4L_dBFxTgr8r_hHUD6RlZEtqaOHDRsFG5PXO5wlvMI0,2324
 runnable/tasks.py,sha256=7yuoeG4ZqfxFUmN4mPS4i6kbQmzEpAwbPQweAUWY-ic,31366
 runnable/utils.py,sha256=amHW3KR_NGTDysGHcSafhh5WJUX7GPBSxqdPyzAIhao,11350
-runnable-0.36.1.dist-info/METADATA,sha256=ajaxCIJIA4TdxVRRL-ja55axuPs2cqDvlma3JQHrweI,10047
-runnable-0.36.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-runnable-0.36.1.dist-info/entry_points.txt,sha256=KkxihZ0LLEiwvFl7RquyqZ0tp2fJDIs7DgzHYDlmc3U,2018
-runnable-0.36.1.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-runnable-0.36.1.dist-info/RECORD,,
+runnable-0.37.0.dist-info/METADATA,sha256=nD9ezWOwkWw3Yi5_NBE_xqjRGKxqf_FnSsYb8P9oqxo,10047
+runnable-0.37.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+runnable-0.37.0.dist-info/entry_points.txt,sha256=KkxihZ0LLEiwvFl7RquyqZ0tp2fJDIs7DgzHYDlmc3U,2018
+runnable-0.37.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+runnable-0.37.0.dist-info/RECORD,,