datatailr 0.1.45__tar.gz → 0.1.47__tar.gz

{datatailr-0.1.45/src/datatailr.egg-info → datatailr-0.1.47}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatailr
-Version: 0.1.45
+Version: 0.1.47
 Summary: Ready-to-Use Platform That Drives Business Insights
 Author-email: Datatailr <info@datatailr.com>
 License-Expression: MIT

{datatailr-0.1.45 → datatailr-0.1.47}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "datatailr"
-version = "0.1.45"
+version = "0.1.47"
 description = "Ready-to-Use Platform That Drives Business Insights"
 readme = "README.md"
 requires-python = ">=3.9"

{datatailr-0.1.45 → datatailr-0.1.47}/src/datatailr/scheduler/base.py

@@ -18,7 +18,7 @@ import tempfile
 import uuid
 from dataclasses import dataclass
 from enum import Enum
-from typing import Callable, Optional, Tuple, Union
+from typing import Callable, Dict, Optional, Tuple, Union
 
 from datatailr import ACL, Environment, User, is_dt_installed
 from datatailr.wrapper import dt__Job
@@ -26,7 +26,7 @@ from datatailr.scheduler.constants import DEFAULT_TASK_MEMORY, DEFAULT_TASK_CPU
 from datatailr.build.image import Image
 from datatailr.errors import BatchJobError
 from datatailr.logging import DatatailrLogger
-from datatailr.utils import run_shell_command
+from datatailr.utils import run_shell_command, dict_to_env_vars
 
 logger = DatatailrLogger(os.path.abspath(__file__)).get_logger()
 __client__ = dt__Job()
@@ -148,6 +148,7 @@ class Job:
         python_requirements: str = "",
         build_script_pre: str = "",
         build_script_post: str = "",
+        env_vars: Dict[str, str | int | float | bool] = {},
         type: Optional[JobType] = JobType.UNKNOWN,
         entrypoint: Optional[EntryPoint] = None,
         update_existing: bool = False,
@@ -186,6 +187,7 @@ class Job:
         self.type = type if entrypoint is None else entrypoint.type
         self.entrypoint = entrypoint
         self.__id = str(uuid.uuid4())
+        self.__env_vars = env_vars
 
     @property
     def id(self) -> str:
@@ -237,8 +239,12 @@ class Job:
             "memory": self.resources.memory,
             "cpu": self.resources.cpu,
         }
+        if self.type == JobType.EXCEL:
+            if "DATATAILR_LOCAL" not in self.__env_vars:
+                self.__env_vars.update({"DATATAILR_LOCAL": "false"})
         if self.type != JobType.BATCH:
             job_dict["entrypoint"] = str(self.entrypoint) if self.entrypoint else None
+            job_dict["env"] = dict_to_env_vars(self.__env_vars)
         return job_dict
 
     def from_dict(self, job_dict: dict):

{datatailr-0.1.45 → datatailr-0.1.47}/src/datatailr/scheduler/batch.py

@@ -32,7 +32,7 @@ from datatailr.scheduler.base import (
 from datatailr.scheduler.constants import DEFAULT_TASK_CPU, DEFAULT_TASK_MEMORY
 from datatailr.scheduler.arguments_cache import ArgumentsCache
 from datatailr.scheduler.schedule import Schedule
-from datatailr.utils import is_dt_installed
+from datatailr.utils import is_dt_installed, dict_to_env_vars
 
 __DAG_CONTEXT__: contextvars.ContextVar = contextvars.ContextVar("dag_context")
 __ARGUMENTS_CACHE__ = ArgumentsCache()
@@ -97,6 +97,7 @@ class BatchJob:
         dependencies: Sequence[Union[str, BatchJob]] = [],
         dag: Optional[Batch] = get_current_manager(),
         argument_mapping: Dict[str, str] = {},
+        env_vars: Dict[str, str | int | float | bool] = {},
     ):
         self.name = name
         self.entrypoint = entrypoint
@@ -112,6 +113,7 @@ class BatchJob:
         assert all(
             isinstance(dep, int) for dep in self.dependencies
         ), "All dependencies must be integers representing job IDs."
+        self.__env_vars = env_vars
         self.dag.add_job(self)
         self.argument_mapping = argument_mapping or {}
 
@@ -193,6 +195,16 @@ class BatchJob:
     def __setstate__(self, state: dict):
         self.__dict__.update(state)
 
+    def update_env_vars(self, env_vars: Dict[str, str | int | float | bool]) -> None:
+        """
+        Update the environment variables for the BatchJob instance.
+
+        :param env_vars: A dictionary of environment variables to update.
+        """
+        env = env_vars.copy()
+        env.update(self.__env_vars)
+        self.__env_vars = env
+
     def to_dict(self):
         """
         Convert the BatchJob instance to a dictionary representation.
@@ -205,6 +217,7 @@ class BatchJob:
             "memory": self.resources.memory if self.resources else DEFAULT_TASK_MEMORY,
             "cpu": self.resources.cpu if self.resources else DEFAULT_TASK_CPU,
             "depends_on": list(self.dependencies),
+            "env": dict_to_env_vars(self.__env_vars),
         }
 
     def to_json(self):
@@ -308,6 +321,7 @@ class Batch(Job):
         python_requirements: str = "",
         build_script_pre: str = "",
         build_script_post: str = "",
+        env_vars: Dict[str, str | int | float | bool] = {},
     ):
         super().__init__(
             environment=environment,
@@ -327,6 +341,7 @@ class Batch(Job):
         self.__BATCH_JOB_NAMES__: Dict[str, int] = {}
         self.__local_run = local_run
         self.__schedule = schedule
+        self.__env_vars = env_vars
 
     @property
     def next_job_id(self):
@@ -369,6 +384,7 @@ class Batch(Job):
                 f"Function {job.entrypoint.function_name} is defined in a different module: "
                 f"{package_path_to_module} != {image_path_to_module}"
             )
+        job.update_env_vars(self.__env_vars)
         self.__jobs.append(job)
 
     def is_job_in(self, job: BatchJob) -> bool:

{datatailr-0.1.45 → datatailr-0.1.47}/src/datatailr/utils.py

@@ -53,3 +53,15 @@ def run_shell_command(command: str) -> Tuple[str, int]:
     if result.returncode != 0:
         raise RuntimeError(f"Command '{command}' failed with error: {result.stderr}")
     return result.stdout.strip(), result.returncode
+
+
+def dict_to_env_vars(env_vars: dict) -> list:
+    """Convert a dictionary of environment variables to a list format suitable for shell commands.
+
+    Args:
+        env_vars (dict): A dictionary where keys are environment variable names and values are their corresponding values.
+
+    Returns:
+        list: A list of lists, where each inner list contains a key-value pair representing an environment variable.
+    """
+    return [[key, str(value)] for key, value in env_vars.items()]

{datatailr-0.1.45 → datatailr-0.1.47/src/datatailr.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatailr
-Version: 0.1.45
+Version: 0.1.47
 Summary: Ready-to-Use Platform That Drives Business Insights
 Author-email: Datatailr <info@datatailr.com>
 License-Expression: MIT

{datatailr-0.1.45 → datatailr-0.1.47}/src/sbin/datatailr_run.py

@@ -85,15 +85,15 @@ def run_command_as_user(command: str | list, user: str, env_vars: dict):
     """
     Run a command as a specific user with the given environment variables.
     """
-    if isinstance(command, str):
-        command = command.split(" ")
+    if isinstance(command, list):
+        command = " ".join(command)
     env_vars = {
         "PATH": get_env_var("PATH", ""),
         "PYTHONPATH": get_env_var("PYTHONPATH", ""),
     } | env_vars
 
     env_kv = [f"{k}={v}" for k, v in env_vars.items()]
-    argv = ["sudo", "-u", user, "env", *env_kv, "bash", "-lc", *command]
+    argv = ["sudo", "-u", user, "env", *env_kv, "bash", "-lc", command]
 
     try:
         result = subprocess.run(
@@ -147,9 +147,11 @@ def main():
     elif job_type == "excel":
         host = get_env_var("DATATAILR_HOST", "")
         entrypoint = get_env_var("DATATAILR_ENTRYPOINT")
+        local = get_env_var("DATATAILR_LOCAL", False)
         env = {
             "DATATAILR_ENTRYPOINT": entrypoint,
             "DATATAILR_HOST": host,
+            "DATATAILR_LOCAL": local,
         } | env
         run_command_as_user("datatailr_run_excel", user, env)
     elif job_type == "ide":