datatailr 0.1.46__tar.gz → 0.1.48__tar.gz

{datatailr-0.1.46/src/datatailr.egg-info → datatailr-0.1.48}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatailr
-Version: 0.1.46
+Version: 0.1.48
 Summary: Ready-to-Use Platform That Drives Business Insights
 Author-email: Datatailr <info@datatailr.com>
 License-Expression: MIT

{datatailr-0.1.46 → datatailr-0.1.48}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "datatailr"
-version = "0.1.46"
+version = "0.1.48"
 description = "Ready-to-Use Platform That Drives Business Insights"
 readme = "README.md"
 requires-python = ">=3.9"

{datatailr-0.1.46 → datatailr-0.1.48}/src/datatailr/scheduler/arguments_cache.py

@@ -115,10 +115,11 @@ class ArgumentsCache:
         :param job: Name of the job.
         :param result: Result of the batch job.
         """
-        path = f"/tmp/datatailr/batch/results/{batch_run_id}_{job}.pkl"
+        path = f"/tmp/datatailr/batch/results/{batch_run_id}/{job}.pkl"
         if self.use_persistent_cache and isinstance(job, str):
             self._add_to_persistent_cache(path, result)
         else:
+            os.makedirs(os.path.dirname(path), exist_ok=True)
             with open(path, "wb") as f:
                 pickle.dump(result, f)
 
@@ -130,7 +131,7 @@ class ArgumentsCache:
         :param job: Name of the job.
         :return: Result of the batch job.
         """
-        path = f"/tmp/datatailr/batch/results/{batch_run_id}_{job}.pkl"
+        path = f"/tmp/datatailr/batch/results/{batch_run_id}/{job}.pkl"
         if self.use_persistent_cache and isinstance(job, str):
             return self._get_from_persistent_cache(path)
         else:

{datatailr-0.1.46 → datatailr-0.1.48}/src/datatailr/scheduler/base.py

@@ -18,7 +18,7 @@ import tempfile
 import uuid
 from dataclasses import dataclass
 from enum import Enum
-from typing import Callable, Optional, Tuple, Union
+from typing import Callable, Dict, Optional, Tuple, Union
 
 from datatailr import ACL, Environment, User, is_dt_installed
 from datatailr.wrapper import dt__Job
@@ -26,7 +26,7 @@ from datatailr.scheduler.constants import DEFAULT_TASK_MEMORY, DEFAULT_TASK_CPU
 from datatailr.build.image import Image
 from datatailr.errors import BatchJobError
 from datatailr.logging import DatatailrLogger
-from datatailr.utils import run_shell_command
+from datatailr.utils import run_shell_command, dict_to_env_vars
 
 logger = DatatailrLogger(os.path.abspath(__file__)).get_logger()
 __client__ = dt__Job()
@@ -148,6 +148,7 @@ class Job:
         python_requirements: str = "",
         build_script_pre: str = "",
         build_script_post: str = "",
+        env_vars: Dict[str, str | int | float | bool] = {},
         type: Optional[JobType] = JobType.UNKNOWN,
         entrypoint: Optional[EntryPoint] = None,
         update_existing: bool = False,
@@ -186,6 +187,7 @@ class Job:
         self.type = type if entrypoint is None else entrypoint.type
         self.entrypoint = entrypoint
         self.__id = str(uuid.uuid4())
+        self.__env_vars = env_vars
 
     @property
     def id(self) -> str:
@@ -237,8 +239,12 @@ class Job:
             "memory": self.resources.memory,
             "cpu": self.resources.cpu,
         }
+        if self.type == JobType.EXCEL:
+            if "DATATAILR_LOCAL" not in self.__env_vars:
+                self.__env_vars.update({"DATATAILR_LOCAL": "false"})
         if self.type != JobType.BATCH:
             job_dict["entrypoint"] = str(self.entrypoint) if self.entrypoint else None
+            job_dict["env"] = dict_to_env_vars(self.__env_vars)
         return job_dict
 
     def from_dict(self, job_dict: dict):

{datatailr-0.1.46 → datatailr-0.1.48}/src/datatailr/scheduler/batch.py

@@ -32,7 +32,7 @@ from datatailr.scheduler.base import (
 from datatailr.scheduler.constants import DEFAULT_TASK_CPU, DEFAULT_TASK_MEMORY
 from datatailr.scheduler.arguments_cache import ArgumentsCache
 from datatailr.scheduler.schedule import Schedule
-from datatailr.utils import is_dt_installed
+from datatailr.utils import is_dt_installed, dict_to_env_vars
 
 __DAG_CONTEXT__: contextvars.ContextVar = contextvars.ContextVar("dag_context")
 __ARGUMENTS_CACHE__ = ArgumentsCache()
@@ -97,6 +97,7 @@ class BatchJob:
         dependencies: Sequence[Union[str, BatchJob]] = [],
         dag: Optional[Batch] = get_current_manager(),
         argument_mapping: Dict[str, str] = {},
+        env_vars: Dict[str, str | int | float | bool] = {},
     ):
         self.name = name
         self.entrypoint = entrypoint
@@ -112,6 +113,7 @@ class BatchJob:
         assert all(
             isinstance(dep, int) for dep in self.dependencies
         ), "All dependencies must be integers representing job IDs."
+        self.__env_vars = env_vars
         self.dag.add_job(self)
         self.argument_mapping = argument_mapping or {}
 
@@ -193,6 +195,16 @@ class BatchJob:
     def __setstate__(self, state: dict):
         self.__dict__.update(state)
 
+    def update_env_vars(self, env_vars: Dict[str, str | int | float | bool]) -> None:
+        """
+        Update the environment variables for the BatchJob instance.
+
+        :param env_vars: A dictionary of environment variables to update.
+        """
+        env = env_vars.copy()
+        env.update(self.__env_vars)
+        self.__env_vars = env
+
     def to_dict(self):
         """
         Convert the BatchJob instance to a dictionary representation.
@@ -205,6 +217,7 @@ class BatchJob:
             "memory": self.resources.memory if self.resources else DEFAULT_TASK_MEMORY,
             "cpu": self.resources.cpu if self.resources else DEFAULT_TASK_CPU,
             "depends_on": list(self.dependencies),
+            "env": dict_to_env_vars(self.__env_vars),
         }
 
     def to_json(self):
@@ -308,6 +321,7 @@ class Batch(Job):
         python_requirements: str = "",
         build_script_pre: str = "",
         build_script_post: str = "",
+        env_vars: Dict[str, str | int | float | bool] = {},
     ):
         super().__init__(
             environment=environment,
@@ -327,6 +341,7 @@ class Batch(Job):
         self.__BATCH_JOB_NAMES__: Dict[str, int] = {}
         self.__local_run = local_run
         self.__schedule = schedule
+        self.__env_vars = env_vars
 
     @property
     def next_job_id(self):
@@ -369,6 +384,7 @@ class Batch(Job):
                 f"Function {job.entrypoint.function_name} is defined in a different module: "
                 f"{package_path_to_module} != {image_path_to_module}"
             )
+        job.update_env_vars(self.__env_vars)
         self.__jobs.append(job)
 
     def is_job_in(self, job: BatchJob) -> bool:

{datatailr-0.1.46 → datatailr-0.1.48}/src/datatailr/utils.py

@@ -53,3 +53,15 @@ def run_shell_command(command: str) -> Tuple[str, int]:
     if result.returncode != 0:
         raise RuntimeError(f"Command '{command}' failed with error: {result.stderr}")
     return result.stdout.strip(), result.returncode
+
+
+def dict_to_env_vars(env_vars: dict) -> list:
+    """Convert a dictionary of environment variables to a list format suitable for shell commands.
+
+    Args:
+        env_vars (dict): A dictionary where keys are environment variable names and values are their corresponding values.
+
+    Returns:
+        list: A list of lists, where each inner list contains a key-value pair representing an environment variable.
+    """
+    return [[key, str(value)] for key, value in env_vars.items()]

{datatailr-0.1.46 → datatailr-0.1.48/src/datatailr.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatailr
-Version: 0.1.46
+Version: 0.1.48
 Summary: Ready-to-Use Platform That Drives Business Insights
 Author-email: Datatailr <info@datatailr.com>
 License-Expression: MIT

{datatailr-0.1.46 → datatailr-0.1.48}/src/sbin/datatailr_run.py

@@ -96,14 +96,11 @@ def run_command_as_user(command: str | list, user: str, env_vars: dict):
     argv = ["sudo", "-u", user, "env", *env_kv, "bash", "-lc", command]
 
     try:
-        result = subprocess.run(
-            argv,
-            check=True,  # raises if returncode != 0
-            capture_output=True,
-            text=True,
-        )
-        logger.info(f"stdout: {result.stdout}")
-        logger.debug(f"stderr: {result.stderr}")
+        proc = subprocess.Popen(argv)
+        returncode = proc.wait()
+        if returncode != 0:
+            logger.error(f"Command failed with exit code {returncode}")
+            sys.exit(returncode)
     except subprocess.CalledProcessError as e:
         logger.error(f"Command failed with exit code {e.returncode}")
         logger.error(f"stderr: {e.stderr}")