runnable 0.35.0__py3-none-any.whl → 0.36.0__py3-none-any.whl

runnable/utils.py

@@ -11,17 +11,16 @@ from collections import OrderedDict
 from datetime import datetime
 from pathlib import Path
 from string import Template as str_template
-from typing import TYPE_CHECKING, Any, Dict, Mapping, Tuple, Union
+from typing import TYPE_CHECKING, Any, Dict, Tuple, Union
 
 from ruamel.yaml import YAML
-from stevedore import driver
 
 import runnable.context as context
 from runnable import console, defaults, names
-from runnable.defaults import TypeMapVariable
+from runnable.defaults import MapVariableType
 
 if TYPE_CHECKING:  # pragma: no cover
-    from runnable.nodes import BaseNode
+    pass
 
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
@@ -66,6 +65,7 @@ def safe_make_dir(directory: Union[str, Path]):
     Path(directory).mkdir(parents=True, exist_ok=True)
 
 
+# TODO: remove this
 def generate_run_id(run_id: str = "") -> str:
     """Generate a new run_id.
 
@@ -147,19 +147,6 @@ def get_module_and_attr_names(command: str) -> Tuple[str, str]:
     return module, func
 
 
-def get_dag_hash(dag: Dict[str, Any]) -> str:
-    """Generates the hash of the dag definition.
-
-    Args:
-        dag (dict): The dictionary object containing the dag definition
-
-    Returns:
-        str: The hash of the dag definition
-    """
-    dag_str = json.dumps(dag, sort_keys=True, ensure_ascii=True)
-    return hashlib.sha1(dag_str.encode("utf-8")).hexdigest()
-
-
 def load_yaml(file_path: str, load_type: str = "safe") -> Dict[str, Any]:
     """Loads an yaml and returns the dictionary.
 
@@ -314,29 +301,6 @@ def remove_prefix(text: str, prefix: str) -> str:
     return text  # or whatever is given
 
 
-def get_tracked_data() -> Dict[str, str]:
-    """Scans the environment variables to find any user tracked variables that have a prefix runnable_TRACK_
-    Removes the environment variable to prevent any clashes in the future steps.
-
-    Returns:
-        dict: A dictionary of user tracked data
-    """
-    tracked_data = {}
-    for env_var, value in os.environ.items():
-        if env_var.startswith(defaults.TRACK_PREFIX):
-            key = remove_prefix(env_var, defaults.TRACK_PREFIX)
-            try:
-                tracked_data[key.lower()] = json.loads(value)
-            except json.decoder.JSONDecodeError:
-                logger.warning(
-                    f"Tracker {key} could not be JSON decoded, adding the literal value"
-                )
-                tracked_data[key.lower()] = value
-
-            del os.environ[env_var]
-    return tracked_data
-
-
 def diff_dict(d1: Dict[str, Any], d2: Dict[str, Any]) -> Dict[str, Any]:
     """
     Given two dicts d1 and d2, return a new dict that has upsert items from d1.
@@ -359,25 +323,6 @@ def diff_dict(d1: Dict[str, Any], d2: Dict[str, Any]) -> Dict[str, Any]:
     return diff
 
 
-# def hash_bytestr_iter(bytesiter, hasher, ashexstr=True):  # pylint: disable=C0116
-#     """Hashes the given bytesiter using the given hasher."""
-#     for block in bytesiter:  # pragma: no cover
-#         hasher.update(block)
-#     return hasher.hexdigest() if ashexstr else hasher.digest()  # pragma: no cover
-
-
-# def file_as_blockiter(afile, blocksize=65536):  # pylint: disable=C0116
-#     """From a StackOverflow answer: that is used to generate a MD5 hash of a large files.
-#     # https://stackoverflow.com/questions/3431825/generating-an-md5-checksum-of-a-file.
-
-#     """
-#     with afile:  # pragma: no cover
-#         block = afile.read(blocksize)
-#         while len(block) > 0:
-#             yield block
-#             block = afile.read(blocksize)
-
-
 def get_data_hash(file_name: str) -> str:
     """Returns the hash of the data file.
 
@@ -397,193 +342,7 @@ def get_data_hash(file_name: str) -> str:
     return file_hash.hexdigest()
 
 
-# TODO: This is not the right place for this.
-def get_node_execution_command(
-    node: BaseNode,
-    map_variable: TypeMapVariable = None,
-    over_write_run_id: str = "",
-    log_level: str = "",
-) -> str:
-    """A utility function to standardize execution call to a node via command line.
-
-    Args:
-        executor (object): The executor class.
-        node (object): The Node to execute
-        map_variable (str, optional): If the node belongs to a map step. Defaults to None.
-
-    Returns:
-        str: The execution command to run a node via command line.
-    """
-    run_id = context.run_context.run_id
-
-    if over_write_run_id:
-        run_id = over_write_run_id
-
-    log_level = log_level or logging.getLevelName(logger.getEffectiveLevel())
-
-    action = (
-        f"runnable execute-single-node {run_id} "
-        f"{context.run_context.pipeline_file} "
-        f"{node._command_friendly_name()} "
-        f"--log-level {log_level} "
-    )
-
-    if context.run_context.from_sdk:
-        action = action + "--mode python "
-
-    if map_variable:
-        action = action + f"--map-variable '{json.dumps(map_variable)}' "
-
-    if context.run_context.configuration_file:
-        action = action + f"--config {context.run_context.configuration_file} "
-
-    if context.run_context.parameters_file:
-        action = action + f"--parameters-file {context.run_context.parameters_file} "
-
-    if context.run_context.tag:
-        action = action + f"--tag {context.run_context.tag}"
-
-    return action
-
-
-# TODO: This is not the right place for this.
-def get_fan_command(
-    mode: str,
-    node: BaseNode,
-    run_id: str,
-    map_variable: TypeMapVariable = None,
-    log_level: str = "",
-) -> str:
-    """
-    An utility function to return the fan "in or out" command
-
-    Args:
-        executor (BaseExecutor): The executor class
-        mode (str): in or out
-        node (BaseNode): The composite node that we are fanning in or out
-        run_id (str): The run id.
-        map_variable (dict, optional): If the node is a map, we have the map variable. Defaults to None.
-
-    Returns:
-        str: The fan in or out command
-    """
-    log_level = log_level or logging.getLevelName(logger.getEffectiveLevel())
-    action = (
-        f"runnable fan {run_id} "
-        f"{node._command_friendly_name()} "  # step name
-        f"{context.run_context.pipeline_file} "  # yaml or python
-        f"{mode} "  # in or out
-        f"--log-level {log_level} "
-    )
-    if context.run_context.configuration_file:
-        action = action + f" --config-file {context.run_context.configuration_file} "
-
-    if context.run_context.parameters_file:
-        action = action + f" --parameters-file {context.run_context.parameters_file}"
-
-    if map_variable:
-        action = action + f" --map-variable '{json.dumps(map_variable)}'"
-
-    if context.run_context.from_sdk:  # execution mode
-        action = action + " --mode python "
-
-    return action
-
-
-# TODO: This is not the right place for this.
-def get_job_execution_command(over_write_run_id: str = "") -> str:
-    """Get the execution command to run a job via command line.
-
-    This function should be used by all executors to submit jobs in remote environment
-    """
-
-    run_id = context.run_context.run_id
-
-    if over_write_run_id:
-        run_id = over_write_run_id
-
-    log_level = logging.getLevelName(logger.getEffectiveLevel())
-
-    action = (
-        f"runnable execute-job {context.run_context.job_definition_file} {run_id} "
-        f" --log-level {log_level}"
-    )
-
-    if context.run_context.configuration_file:
-        action = action + f" --config {context.run_context.configuration_file}"
-
-    if context.run_context.parameters_file:
-        action = action + f" --parameters {context.run_context.parameters_file}"
-
-    if context.run_context.from_sdk:
-        action = action + " --mode python "
-
-    if context.run_context.tag:
-        action = action + f" --tag {context.run_context.tag}"
-
-    return action
-
-
-def get_provider_by_name_and_type(
-    service_type: str, service_details: defaults.ServiceConfig
-):
-    """Given a service type, one of executor, run_log_store, catalog, secrets and the config
-    return the exact child class implementing the service.
-    We use stevedore to do the work for us.
-
-    Args:
-        service_type (str): One of executor, run_log_store, catalog, secrets
-        service_details (dict): The config used to instantiate the service.
-
-    Raises:
-        Exception: If the service by that name does not exist
-
-    Returns:
-        object: A service object
-    """
-
-    namespace = service_type
-
-    service_name = service_details["type"]
-    service_config: Mapping[str, Any] = {}
-    if "config" in service_details:
-        service_config = service_details.get("config", {})
-
-    logger.debug(
-        f"Trying to get a service of {service_type} of the name {service_name} with config: {service_config}"
-    )
-    try:
-        mgr = driver.DriverManager(
-            namespace=namespace,
-            name=service_name,
-            invoke_on_load=True,
-            invoke_kwds={**service_config},
-        )
-        return mgr.driver
-    except Exception as _e:
-        logger.exception(
-            f"Could not find the service of type: {service_type} with config: {service_details}"
-        )
-        raise Exception(
-            f"Could not find the service of type: {service_type} with config: {service_details}"
-        ) from _e
-
-
-def get_run_config() -> dict:
-    """Given an executor with assigned services, return the run_config.
-
-    Args:
-        executor (object): The executor with all the services assigned.
-
-    Returns:
-        dict: The run_config.
-    """
-
-    run_config = context.run_context.model_dump(by_alias=True)
-    return run_config
-
-
-def json_to_ordered_dict(json_str: str) -> TypeMapVariable:
+def json_to_ordered_dict(json_str: str) -> MapVariableType:
     """Decode a JSON str into OrderedDict.
 
     Args:
@@ -598,27 +357,6 @@ def json_to_ordered_dict(json_str: str) -> TypeMapVariable:
     return OrderedDict()
 
 
-def set_runnable_environment_variables(
-    run_id: str = "", configuration_file: str = "", tag: str = ""
-) -> None:
-    """Set the environment variables used by runnable. This function should be called during the prepare configurations
-    by all executors.
-
-    Args:
-        run_id (str, optional): The run id of the execution. Defaults to None.
-        configuration_file (str, optional): The configuration file if used. Defaults to None.
-        tag (str, optional): The tag associated with a run. Defaults to None.
-    """
-    if run_id:
-        os.environ[defaults.ENV_RUN_ID] = run_id
-
-    if configuration_file:
-        os.environ[defaults.RUNNABLE_CONFIG_FILE] = configuration_file
-
-    if tag:
-        os.environ[defaults.RUNNABLE_RUN_TAG] = tag
-
-
 def gather_variables() -> Dict[str, str]:
     """Gather all the environment variables used by runnable. All the variables start with runnable_VAR_.
 
@@ -635,7 +373,7 @@ def gather_variables() -> Dict[str, str]:
     return variables
 
 
-def make_log_file_name(name: str, map_variable: TypeMapVariable) -> str:
+def make_log_file_name(name: str, map_variable: MapVariableType) -> str:
     random_tag = "".join(random.choices(string.ascii_uppercase + string.digits, k=3))
     log_file_name = name
 

{runnable-0.35.0.dist-info → runnable-0.36.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: runnable
-Version: 0.35.0
+Version: 0.36.0
 Summary: Add your description here
 Author-email: "Vammi, Vijay" <vijay.vammi@astrazeneca.com>
 License-File: LICENSE

runnable-0.36.0.dist-info/RECORD

@@ -0,0 +1,74 @@
+extensions/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+extensions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+extensions/catalog/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+extensions/catalog/any_path.py,sha256=atB5gWPRX6ptW6zwYeCVb_fh0qhs7WAFO9HIsnMZl98,7350
+extensions/catalog/file_system.py,sha256=T_qFPFfrmykoAMc1rjNi_DBb437me8WPRcFglwAK744,1767
+extensions/catalog/minio.py,sha256=R3GvfCxN1GTcs4bQIAWh79_GHDTVd14gnpKlzwFeKUI,2363
+extensions/catalog/pyproject.toml,sha256=lLNxY6v04c8I5QK_zKw_E6sJTArSJRA_V-79ktaA3Hk,279
+extensions/catalog/s3.py,sha256=Sw5t8_kVRprn3uGGJCiHn7M9zw1CLaCOFj6YErtfG0o,287
+extensions/job_executor/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+extensions/job_executor/__init__.py,sha256=5_pKw7GJbwmXL-QPdZwtfRKVHuKiPqLheK3rGU5G7-M,5813
+extensions/job_executor/emulate.py,sha256=d9KI4If0DeRkKv9iCSKyZCziK_3ARsJvs5RUFo9yigc,3220
+extensions/job_executor/k8s.py,sha256=W14u1xwJDtb-gyGdNl62IwWIs49AUO2oTAAxQCm6YT0,16456
+extensions/job_executor/k8s_job_spec.yaml,sha256=7aFpxHdO_p6Hkc3YxusUOuAQTD1Myu0yTPX9DrhxbOg,1158
+extensions/job_executor/local.py,sha256=3ZbCFXBvbLlMp10JTmQJJrjBKG2keHI6SH8hEvmHDkA,2230
+extensions/job_executor/local_container.py,sha256=uoRbsyR5QiVytWJQtF7nXUA1h8wHAaQezedWFKl3btM,6914
+extensions/job_executor/pyproject.toml,sha256=UIEgiCYHTXcRWSByNMFuKJFKgxTBpQqTqyUecIsb_Vc,286
+extensions/nodes/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+extensions/nodes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+extensions/nodes/conditional.py,sha256=lsXLerIDWJk0fHo4EEngl8rYw6dN40EgKIgdBuvt8ro,8641
+extensions/nodes/fail.py,sha256=P_wryruE2INiAmu4Vf7q4fTEkfbIMnhT71Y9IrOqRkM,2170
+extensions/nodes/map.py,sha256=s0l07uAc6j8MO7Fqch0DzOE3_CsV6pBNHj5E3nf1S60,13659
+extensions/nodes/parallel.py,sha256=cuxpR1e0KwuxvoZaSwiiEn9YmYskplbjLBWBzzU-Ti8,5952
+extensions/nodes/pyproject.toml,sha256=YTu-ETN3JNFSkMzzWeOwn4m-O2nbRH-PmiPBALDCUw4,278
+extensions/nodes/stub.py,sha256=o9DjBekNa9O4b0VQOiNOA9eNjJ3C2a9Sn9d2fX7KaWg,2715
+extensions/nodes/success.py,sha256=yT4WkUI-1YN6qy4Ji6zSoZFXI9jOl6Ond4WxZRq-33k,2179
+extensions/nodes/task.py,sha256=m5_s5u7lQDoTmeO6CG94btrG5If9AKm2-RvOzvm0fDc,3147
+extensions/pipeline_executor/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+extensions/pipeline_executor/__init__.py,sha256=u1wLhQfDB1WotCoLIg1mwGlJTupfV312xACwjUxsbJw,24521
+extensions/pipeline_executor/argo.py,sha256=Jz0sA3rNiG_JrCEMCA4b7aGjpWo8erQFLgymgCXrGWU,38349
+extensions/pipeline_executor/emulate.py,sha256=ziaIzEpYgVOm2PPCY8bnRdoevMwS8OY9aEUXpwCFQR0,3547
+extensions/pipeline_executor/local.py,sha256=_8dtj88mI3xeZT4UPwtHNtbyi_VVzz_lpIxvC0UZS-g,1887
+extensions/pipeline_executor/local_container.py,sha256=oJsd7Pcxur2ZzaBqTN6IUk_ZnMqYkxrK65BVKo_2P2k,10262
+extensions/pipeline_executor/mocked.py,sha256=myaU5Jy4LRRwe2-fBiX5Ts-1xc1a6nqXHxQlmur5H-M,5679
+extensions/pipeline_executor/pyproject.toml,sha256=ykTX7srR10PBYb8LsIwEj8vIPPIEZQ5V_R7VYbZ-ido,291
+extensions/pipeline_executor/retry.py,sha256=sBfZiS9ryw_hNCMAza82mXyV1wPjOJb20EEF9f5FUng,6790
+extensions/run_log_store/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+extensions/run_log_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+extensions/run_log_store/any_path.py,sha256=0nN_LHbm2W6AHkerQmsVHq3EoybFQF8lxpCicacHo8Y,2861
+extensions/run_log_store/chunked_fs.py,sha256=wHMKcAx6uFI4OOTp7QWCdGq9WvEFesbLp9VxHZU28l0,3341
+extensions/run_log_store/chunked_minio.py,sha256=Itfkw4Ycf0uLCqxH3Uk_itmVgT7ipJp05yKfD22WBiY,4007
+extensions/run_log_store/file_system.py,sha256=hhrbhSnuzv8yzBr6DAu45NT8-sawPP86WA2-LY70vjw,2781
+extensions/run_log_store/generic_chunked.py,sha256=EnhRxlqm1jG-Tdxul4sY8OeCX5fK9FY2v8DZanX9-5o,20455
+extensions/run_log_store/minio.py,sha256=omrKDSdRzmnVBg9xXkkdQb-icBIgBDRdpmwGRlMyCGk,3453
+extensions/run_log_store/pyproject.toml,sha256=YnmXsFvFG9uv_c0spLYBsNI_1sbktqxtHsOuClyvZ3g,288
+extensions/run_log_store/db/implementation_FF.py,sha256=euTnh0xzNF0e_DyfHQ4W-kG1AwTr8u7OuO3_cZkR5bM,5237
+extensions/run_log_store/db/integration_FF.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+extensions/secrets/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+extensions/secrets/dotenv.py,sha256=nADHXI6KJ_LUYOIe5EbtYH-21OBebSNVr0Pjb1GlZ7w,1573
+extensions/secrets/pyproject.toml,sha256=mLJNImNcBlbLKHh-0ugVWT9V83R4RibyyYDtBCSqVF4,282
+extensions/tasks/torch.py,sha256=H1b84vV1j8a8U0ddabPEOcCVUWZHXx7FUbJDJ0PxzuI,10283
+extensions/tasks/torch_config.py,sha256=UjfMitT-TXASRDGR30I2vDRnyk7JQnR-5CsOVidjpSY,2833
+runnable/__init__.py,sha256=LUruVUZaLLZC4Xz-6_w_0qQ-OtxMZR5VVIPJeOxLRUY,538
+runnable/catalog.py,sha256=A03iMLW8S6_AZjucBoFhl7fPKx-khTLQeICVAUjd9WU,3767
+runnable/cli.py,sha256=CziCKBoL-dklSEo_x-YO1AysrG2eaf2LMQZbcNTeCbM,7283
+runnable/context.py,sha256=yEJodj6hbBAcvpQdoAUAwoYXsqiJTQtxmGTJWIaexHs,17515
+runnable/datastore.py,sha256=2pYg4i1JRMzw_CUUIsPOWt7wYPiGBamfo-CPVAkEH54,32375
+runnable/defaults.py,sha256=4UYuShnjEyWP529UlFnubvkBpOcczKIdE4jEOhPBwl4,3076
+runnable/entrypoints.py,sha256=46prgr3_FYtBMlRbUXIDSpgZUBgaxcdJAekXhgEIj7M,6578
+runnable/exceptions.py,sha256=t5tSlYqe_EjU5liXu32yLLh_yrnXeFL93BuXfmQzV98,3268
+runnable/executor.py,sha256=CwzHkeGVpocACZLzfFS94TzKeiaPLv4NtXtvT3eoocY,15222
+runnable/graph.py,sha256=poQz5zcvq89ju_u5sYlunQLPbHnXTaUmjcvstPwvT4U,16536
+runnable/names.py,sha256=A9ldUyULXuWjJ1MoXihHqlg-xeTVX-oWYTO5Ah0trmo,8128
+runnable/nodes.py,sha256=JXB7TLA5zrFFk1cJXLrSBfunp4aVRFw_Y5uIjIGOpHg,16805
+runnable/parameters.py,sha256=zEehAliVvCOLOnNZ4ExJvSDJM_2PWY0URZ0bmZUgCQA,5289
+runnable/pickler.py,sha256=ydJ_eti_U1F4l-YacFp7BWm6g5vTn04UXye25S1HVok,2684
+runnable/sdk.py,sha256=reP1TEy05PAjQdHHVobAbt1bqVkSDSqPENGp173Ak7s,33434
+runnable/secrets.py,sha256=4L_dBFxTgr8r_hHUD6RlZEtqaOHDRsFG5PXO5wlvMI0,2324
+runnable/tasks.py,sha256=TUNcFCuu3B0E223EC9z1nx4J2K_P1vkLg7wHIxEuepo,33720
+runnable/utils.py,sha256=amHW3KR_NGTDysGHcSafhh5WJUX7GPBSxqdPyzAIhao,11350
+runnable-0.36.0.dist-info/METADATA,sha256=c4n_9b8IfivaF8SzYELhRFEjRqcE257EYDZNdOSOioY,10166
+runnable-0.36.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+runnable-0.36.0.dist-info/entry_points.txt,sha256=KkxihZ0LLEiwvFl7RquyqZ0tp2fJDIs7DgzHYDlmc3U,2018
+runnable-0.36.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+runnable-0.36.0.dist-info/RECORD,,

{runnable-0.35.0.dist-info → runnable-0.36.0.dist-info}/entry_points.txt

@@ -8,6 +8,7 @@ minio = extensions.catalog.minio:MinioCatalog
 s3 = extensions.catalog.s3:S3Catalog
 
 [job_executor]
+emulator = extensions.job_executor.emulate:EmulatorJobExecutor
 k8s-job = extensions.job_executor.k8s:K8sJobExecutor
 local = extensions.job_executor.local:LocalJobExecutor
 local-container = extensions.job_executor.local_container:LocalContainerJobExecutor
@@ -15,19 +16,19 @@ mini-k8s-job = extensions.job_executor.k8s:MiniK8sJobExecutor
 
 [nodes]
 conditional = extensions.nodes.conditional:ConditionalNode
-dag = extensions.nodes.nodes:DagNode
-fail = extensions.nodes.nodes:FailNode
-map = extensions.nodes.nodes:MapNode
-parallel = extensions.nodes.nodes:ParallelNode
-stub = extensions.nodes.nodes:StubNode
-success = extensions.nodes.nodes:SuccessNode
-task = extensions.nodes.nodes:TaskNode
+fail = extensions.nodes.fail:FailNode
+map = extensions.nodes.map:MapNode
+parallel = extensions.nodes.parallel:ParallelNode
+stub = extensions.nodes.stub:StubNode
+success = extensions.nodes.success:SuccessNode
+task = extensions.nodes.task:TaskNode
 
 [pickler]
 pickle = runnable.pickler:NativePickler
 
 [pipeline_executor]
 argo = extensions.pipeline_executor.argo:ArgoExecutor
+emulator = extensions.pipeline_executor.emulate:Emulator
 local = extensions.pipeline_executor.local:LocalExecutor
 local-container = extensions.pipeline_executor.local_container:LocalContainerExecutor
 mocked = extensions.pipeline_executor.mocked:MockedExecutor