datatailr 0.1.67__tar.gz → 0.1.69__tar.gz

{datatailr-0.1.67/src/datatailr.egg-info → datatailr-0.1.69}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatailr
-Version: 0.1.67
+Version: 0.1.69
 Summary: Ready-to-Use Platform That Drives Business Insights
 Author-email: Datatailr <info@datatailr.com>
 License-Expression: MIT

{datatailr-0.1.67 → datatailr-0.1.69}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "datatailr"
-version = "0.1.67"
+version = "0.1.69"
 description = "Ready-to-Use Platform That Drives Business Insights"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -57,6 +57,10 @@ dev = [
     "myst-parser"
 ]
 
+[tool.coverage.run]
+branch = true
+source = ["./src/datatailr"]
+
 [tool.ruff]
 src = [
     "src",

{datatailr-0.1.67 → datatailr-0.1.69}/src/datatailr/acl.py

@@ -25,15 +25,15 @@ class ACL:
         self,
         user: Union[User, str],
         group: Optional[Union[Group, str]] = None,
-        permissions: Optional[List[str]] = None,
+        permissions: Optional[List[str] | str] = None,
     ):
         if user is None:
             user = User.signed_user()
         self.user = user if isinstance(user, User) else User.get(user)
-        if self.user is not None:
-            self.group = (
-                group if group and isinstance(group, Group) else self.user.primary_group
-            )
+        if group is None:
+            group = self.user.primary_group
+        group = group if isinstance(group, Group) else Group.get(str(group))
+        self.group = group
         self.permissions = permissions or "rwr---"
 
         self.__group_can_read = False

{datatailr-0.1.67 → datatailr-0.1.69}/src/datatailr/scheduler/arguments_cache.py

@@ -28,6 +28,7 @@ from typing import Any, Dict, Optional
 
 from datatailr import is_dt_installed, Blob
 from datatailr.errors import DatatailrError
+from datatailr.wrapper import dt__Tag
 
 
 __BLOB_STORAGE__ = Blob()
@@ -48,11 +49,12 @@ class ArgumentsCache:
 
         :param use_persistent_cache: If True, use the persistent cache backend. Otherwise, use in-memory cache.
         """
+        self.__bucket_name__ = dt__Tag().get("blob_storage_prefix") + "batch"
         self.use_persistent_cache = use_persistent_cache
         if not self.use_persistent_cache:
             # Create a temp folder, for local caching
-            os.makedirs("/tmp/datatailr/batch/arguments", exist_ok=True)
-            os.makedirs("/tmp/datatailr/batch/results", exist_ok=True)
+            os.makedirs(f"/tmp/{self.__bucket_name__}/arguments", exist_ok=True)
+            os.makedirs(f"/tmp/{self.__bucket_name__}/results", exist_ok=True)
 
     def add_arguments(self, batch_id: str, arguments: Dict[str, Any]):
         """
@@ -62,7 +64,7 @@ class ArgumentsCache:
         :param job_name: Name of the job.
         :param arguments: Dictionary of arguments to store.
         """
-        path = f"/tmp/datatailr/batch/arguments/{batch_id}.pkl"
+        path = f"/tmp/{self.__bucket_name__}/arguments/{batch_id}.pkl"
         if self.use_persistent_cache:
             self._add_to_persistent_cache(path, arguments)
         else:
@@ -79,7 +81,7 @@ class ArgumentsCache:
         :param job_name: Name of the job.
         :return: Dictionary of arguments.
         """
-        path = f"/tmp/datatailr/batch/arguments/{batch_id}.pkl"
+        path = f"/tmp/{self.__bucket_name__}/arguments/{batch_id}.pkl"
         if self.use_persistent_cache and isinstance(job, str):
             try:
                 arg_keys = self._get_from_persistent_cache(path)
@@ -115,7 +117,7 @@ class ArgumentsCache:
         :param job: Name of the job.
         :param result: Result of the batch job.
         """
-        path = f"/tmp/datatailr/batch/results/{batch_run_id}/{job}.pkl"
+        path = f"/tmp/{self.__bucket_name__}/results/{batch_run_id}/{job}.pkl"
         if self.use_persistent_cache and isinstance(job, str):
             self._add_to_persistent_cache(path, result)
         else:
@@ -131,7 +133,7 @@ class ArgumentsCache:
         :param job: Name of the job.
         :return: Result of the batch job.
         """
-        path = f"/tmp/datatailr/batch/results/{batch_run_id}/{job}.pkl"
+        path = f"/tmp/{self.__bucket_name__}/results/{batch_run_id}/{job}.pkl"
         if self.use_persistent_cache and isinstance(job, str):
             return self._get_from_persistent_cache(path)
         else:

{datatailr-0.1.67 → datatailr-0.1.69}/src/datatailr/scheduler/base.py

@@ -290,14 +290,30 @@ class Job:
         Returns a tuple of (branch: str, commit_hash: str).
         """
         path_to_repo = self.image.path_to_repo or "."
-        local_commit = run_shell_command(f"cd {path_to_repo} && git rev-parse HEAD")[0]
-        branch_name = run_shell_command(
-            f"cd {path_to_repo} && git rev-parse --abbrev-ref HEAD"
-        )[0]
+        try:
+            local_commit = run_shell_command(
+                f"cd {path_to_repo} && git rev-parse HEAD"
+            )[0]
+            branch_name = run_shell_command(
+                f"cd {path_to_repo} && git rev-parse --abbrev-ref HEAD"
+            )[0]
+            return_code = run_shell_command(
+                f"cd {path_to_repo} && git diff --exit-code"
+            )
+        except RuntimeError as e:
+            if "git: not found" in str(e):
+                logger.warning(
+                    "Git is not installed or not found in PATH. Repository validation is not possible."
+                )
+                branch_name, local_commit, return_code = "unknown", "unknown", None
+            else:
+                raise RepoValidationError(
+                    f"Error accessing git repository at {path_to_repo}: {e}"
+                ) from e
 
         if os.getenv("DATATAILR_ALLOW_UNSAFE_SCHEDULING", "false").lower() == "true":
             return branch_name, local_commit
-        return_code = run_shell_command(f"cd {path_to_repo} && git diff --exit-code")
+
         is_committed = return_code is not None and return_code[1] == 0
 
         if not is_committed:
@@ -337,6 +353,13 @@ class Job:
         """
         return {}
 
+    def __set_existing_id__(self, job_id: dict):
+        if isinstance(job_id, dict) and "Ids" in job_id and len(job_id["Ids"]) > 0:
+            job_id = job_id["Ids"][0]
+            if isinstance(job_id, str):
+                logger.debug(f"Setting existing job ID: {job_id}")
+                self.__id = job_id
+
     def __run_command__(self, command: str) -> Tuple[bool, str]:
         """
         Run a command in the context of the job.
@@ -350,18 +373,23 @@ class Job:
             temp_file_name = self.__prepare__()
 
             if command == "run":
-                __client__.run(f"file://{temp_file_name}", **self.get_schedule_args())
+                result = __client__.run(
+                    f"file://{temp_file_name}", **self.get_schedule_args()
+                )
             elif command == "save":
-                __client__.save(f"file://{temp_file_name}", **self.get_schedule_args())
+                result = __client__.save(
+                    f"file://{temp_file_name}", **self.get_schedule_args()
+                )
             elif command == "start":
-                __client__.start(self.name, environment=self.environment)
+                result = __client__.start(self.name, environment=self.environment)
             else:
                 raise ValueError(f"Unknown command: {command}")
             os.remove(temp_file_name)
         except Exception as e:
             logger.error(f"Error running command '{command}': {e}")
             return False, str(e)
-        return True, f"Job '{self.name}' {command}d successfully."
+        self.__set_existing_id__(result)
+        return True, result
 
     def save(self) -> Tuple[bool, str]:
         """

{datatailr-0.1.67 → datatailr-0.1.69}/src/datatailr/scheduler/batch.py

@@ -501,8 +501,10 @@ class Batch(Job):
         __ARGUMENTS_CACHE__.add_arguments(self.id, args)
 
     def save(self) -> Tuple[bool, str]:
-        self.prepare_args()
-        return super().save()
+        status = super().save()
+        if status[0]:
+            self.prepare_args()
+        return status
 
     def run(self) -> Tuple[bool, str]:
         self.prepare_args()

datatailr-0.1.69/src/datatailr/tag.py

@@ -0,0 +1,35 @@
+# *************************************************************************
+#
+#  Copyright (c) 2025 - Datatailr Inc.
+#  All Rights Reserved.
+#
+#  This file is part of Datatailr and subject to the terms and conditions
+#  defined in 'LICENSE.txt'. Unauthorized copying and/or distribution
+#  of this file, in parts or full, via any medium is strictly prohibited.
+# *************************************************************************
+
+from typing import Any, ClassVar
+
+
+class dt__Tag:
+    """
+    Tag management for local runs in the absence of the DataTailr platform.
+    All instances share the same tag store.
+    """
+
+    # shared across all instances
+    tags: ClassVar[dict[str, Any]] = {
+        "blob_storage_prefix": "local-no-dt-",
+    }
+
+    def ls(self) -> dict[str, Any]:
+        return self.__class__.tags
+
+    def get(self, name: str) -> Any:
+        return self.__class__.tags.get(name)
+
+    def set(self, name: str, value: Any) -> None:
+        self.__class__.tags[name] = value
+
+    def rm(self, name: str) -> None:
+        self.__class__.tags.pop(name, None)

{datatailr-0.1.67 → datatailr-0.1.69}/src/datatailr/wrapper.py

@@ -14,6 +14,7 @@ import subprocess
 from typing import Union
 
 from datatailr.utils import is_dt_installed
+from datatailr.tag import dt__Tag as local_no_dt_Tag
 
 API_JSON_PATH: Union[str, None] = None
 
@@ -199,7 +200,7 @@ dt__Email = globals().get("dt__Email", mock_cli_tool)
 dt__Kv = globals().get("dt__Kv", mock_cli_tool)
 dt__Log = globals().get("dt__Log", mock_cli_tool)
 dt__Node = globals().get("dt__Node", mock_cli_tool)
-dt__Tag = globals().get("dt__Tag", mock_cli_tool)
+dt__Tag = globals().get("dt__Tag", local_no_dt_Tag)
 dt__Registry = globals().get("dt__Registry", mock_cli_tool)
 dt__Service = globals().get("dt__Service", mock_cli_tool)
 dt__Settings = globals().get("dt__Settings", mock_cli_tool)

{datatailr-0.1.67 → datatailr-0.1.69/src/datatailr.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatailr
-Version: 0.1.67
+Version: 0.1.69
 Summary: Ready-to-Use Platform That Drives Business Insights
 Author-email: Datatailr <info@datatailr.com>
 License-Expression: MIT

{datatailr-0.1.67 → datatailr-0.1.69}/src/datatailr.egg-info/SOURCES.txt

@@ -9,6 +9,7 @@ src/datatailr/dt_json.py
 src/datatailr/errors.py
 src/datatailr/group.py
 src/datatailr/logging.py
+src/datatailr/tag.py
 src/datatailr/user.py
 src/datatailr/utils.py
 src/datatailr/version.py

{datatailr-0.1.67 → datatailr-0.1.69}/src/sbin/datatailr_run.py

@@ -32,11 +32,12 @@
 #     DATATAILR_JOB_ID - the unique identifier for the job.
 
 
+import concurrent.futures
 import subprocess
 import os
-import sys
+import shlex
 import sysconfig
-from typing import Tuple
+from typing import Optional, Tuple
 from datatailr.logging import DatatailrLogger
 from datatailr.utils import is_dt_installed
 
@@ -82,40 +83,90 @@ def create_user_and_group() -> Tuple[str, str]:
     return user, group
 
 
-def run_command_as_user(command: str | list, user: str, env_vars: dict):
-    """
-    Run a command as a specific user with the given environment variables.
-    """
+def prepare_command_argv(command: str | list, user: str, env_vars: dict) -> list[str]:
     if isinstance(command, str):
-        command = command.split(" ")
+        command = shlex.split(command)
 
     python_libdir = sysconfig.get_config_var("LIBDIR")
-
     ld_library_path = get_env_var("LD_LIBRARY_PATH", "")
+
     if ld_library_path:
         python_libdir = ld_library_path + ":" + python_libdir
-    else:
-        python_libdir = python_libdir
 
-    env_vars = {
+    # Base environment variables setup
+    base_env = {
         "PATH": get_env_var("PATH", ""),
         "PYTHONPATH": get_env_var("PYTHONPATH", ""),
         "LD_LIBRARY_PATH": python_libdir,
-    } | env_vars
+    }
+
+    merged_env = base_env | env_vars
+    env_kv = [f"{k}={v}" for k, v in merged_env.items()]
+    return ["sudo", "-u", user, "env", *env_kv, *command]
 
-    env_kv = [f"{k}={v}" for k, v in env_vars.items()]
-    argv = ["sudo", "-u", user, "env", *env_kv, *command]
+
+def run_single_command_non_blocking(
+    command: str | list,
+    user: str,
+    env_vars: dict,
+    log_stream_name: Optional[str | None] = None,
+) -> int:
+    """
+    Runs a single command non-blocking and returns the exit code after it finishes.
+    This is designed to be run within an Executor.
+    """
+    argv = prepare_command_argv(command, user, env_vars)
+    cmd_label = " ".join(argv[4:])  # For logging purposes
 
     try:
-        proc = subprocess.Popen(argv)
+        if log_stream_name:
+            stdout_file_path = f"/opt/datatailr/var/log/{log_stream_name}.log"
+            stderr_file_path = f"/opt/datatailr/var/log/{log_stream_name}_error.log"
+            with (
+                open(stdout_file_path, "ab", buffering=0) as stdout_file,
+                open(stderr_file_path, "ab", buffering=0) as stderr_file,
+            ):
+                proc = subprocess.Popen(argv, stdout=stdout_file, stderr=stderr_file)
+        else:
+            proc = subprocess.Popen(argv)
         returncode = proc.wait()
+
         if returncode != 0:
-            logger.error(f"Command failed with exit code {returncode}")
-            sys.exit(returncode)
-    except subprocess.CalledProcessError as e:
-        logger.error(f"Command failed with exit code {e.returncode}")
-        logger.error(f"stderr: {e.stderr}")
-        sys.exit(1)
+            logger.error(f"Command '{cmd_label}' failed with exit code {returncode}")
+        return returncode
+    except Exception as e:
+        logger.error(f"Execution error for '{cmd_label}': {e}")
+        return 1
+
+
+def run_commands_in_parallel(
+    commands: list[str | list],
+    user: str,
+    env_vars: dict,
+    log_stream_names: Optional[list[str | None]] = None,
+) -> tuple[int, int]:
+    """
+    Executes two commands concurrently using a ThreadPoolExecutor.
+    Returns a tuple of (return_code_cmd1, return_code_cmd2).
+    """
+    with concurrent.futures.ThreadPoolExecutor(max_workers=len(commands)) as executor:
+        futures = []
+        for command, log_stream_name in zip(
+            commands, log_stream_names or [None] * len(commands)
+        ):
+            futures.append(
+                executor.submit(
+                    run_single_command_non_blocking,
+                    command,
+                    user,
+                    env_vars,
+                    log_stream_name,
+                )
+            )
+        results = [
+            future.result() for future in concurrent.futures.as_completed(futures)
+        ]
+        return results[0], results[1]
 
 
 def main():
@@ -137,7 +188,7 @@ def main():
             "DATATAILR_BATCH_ID": batch_id,
             "DATATAILR_BATCH_ENTRYPOINT": entrypoint,
         } | env
-        run_command_as_user("datatailr_run_batch", user, env)
+        run_single_command_non_blocking("datatailr_run_batch", user, env)
     elif job_type == "service":
         port = get_env_var("DATATAILR_SERVICE_PORT", 8080)
         entrypoint = get_env_var("DATATAILR_ENTRYPOINT")
@@ -145,13 +196,13 @@ def main():
             "DATATAILR_ENTRYPOINT": entrypoint,
             "DATATAILR_SERVICE_PORT": port,
         } | env
-        run_command_as_user("datatailr_run_service", user, env)
+        run_single_command_non_blocking("datatailr_run_service", user, env)
     elif job_type == "app":
         entrypoint = get_env_var("DATATAILR_ENTRYPOINT")
         env = {
             "DATATAILR_ENTRYPOINT": entrypoint,
         } | env
-        run_command_as_user("datatailr_run_app", user, env)
+        run_single_command_non_blocking("datatailr_run_app", user, env)
     elif job_type == "excel":
         host = get_env_var("DATATAILR_HOST", "")
         local = get_env_var("DATATAILR_LOCAL", "")
@@ -162,8 +213,13 @@ def main():
             "DATATAILR_HOST": host,
             "DATATAILR_LOCAL": local,
         } | env
-        run_command_as_user("datatailr_run_excel", user, env)
+        run_single_command_non_blocking("datatailr_run_excel", user, env)
     elif job_type == "workspace":
+        # Set a custom PS1 for the IDE terminal: 17:38|user@my-ide/~/dir/path:$
+        env["PS1"] = (
+            r"""\[\e[2m\]\A\[\e[0m\]|\[\e[38;5;40m\]\u\[\e[92m\]@${DATATAILR_JOB_NAME:-datatailr}\[\e[0m\]/\[\e[94;1m\]\w\[\e[0m\]\$"""
+        )
+        os.makedirs("/opt/datatailr/var/log", exist_ok=True)
         ide_command = [
             "code-server",
             "--auth=none",
@@ -178,8 +234,9 @@ def main():
             "--NotebookApp.token=''",
             "--NotebookApp.password=''",
         ]
-        # TODO: run both commands in parallel
-        run_command_as_user(ide_command, user, env)
+        run_commands_in_parallel(
+            [ide_command, jupyter_command], user, env, ["code-server", "jupyter"]
+        )
 
     else:
         raise ValueError(f"Unknown job type: {job_type}")

{datatailr-0.1.67 → datatailr-0.1.69}/src/sbin/datatailr_run_excel.py

@@ -35,6 +35,8 @@ def run():
             raise ValueError("Environment variable 'DATATAILR_HOST' is not set.")
         local_flag = ""
 
-    entrypoint = f'/opt/datatailr/bin/dt-excel.sh -n -H "{hostname}" {local_flag} -p 8080 -w 8000 {entrypoint}'
+    module_name = entrypoint.split(":", 1)[0]
+
+    entrypoint = f'/opt/datatailr/bin/dt-excel.sh -n -H "{hostname}" {local_flag} -p 8080 -w 8000 {module_name}'
     logger.info(f"Running entrypoint: {entrypoint}")
     subprocess.run(entrypoint, shell=True)