datatailr 0.1.67__py3-none-any.whl → 0.1.68__py3-none-any.whl

datatailr/acl.py

@@ -25,15 +25,15 @@ class ACL:
         self,
         user: Union[User, str],
         group: Optional[Union[Group, str]] = None,
-        permissions: Optional[List[str]] = None,
+        permissions: Optional[List[str] | str] = None,
     ):
         if user is None:
             user = User.signed_user()
         self.user = user if isinstance(user, User) else User.get(user)
-        if self.user is not None:
-            self.group = (
-                group if group and isinstance(group, Group) else self.user.primary_group
-            )
+        if group is None:
+            group = self.user.primary_group
+        group = group if isinstance(group, Group) else Group.get(str(group))
+        self.group = group
         self.permissions = permissions or "rwr---"
 
         self.__group_can_read = False

datatailr/sbin/datatailr_run.py

@@ -32,9 +32,10 @@
 #     DATATAILR_JOB_ID - the unique identifier for the job.
 
 
+import concurrent.futures
 import subprocess
 import os
-import sys
+import shlex
 import sysconfig
 from typing import Tuple
 from datatailr.logging import DatatailrLogger
@@ -82,40 +83,67 @@ def create_user_and_group() -> Tuple[str, str]:
     return user, group
 
 
-def run_command_as_user(command: str | list, user: str, env_vars: dict):
-    """
-    Run a command as a specific user with the given environment variables.
-    """
+def prepare_command_argv(command: str | list, user: str, env_vars: dict) -> list[str]:
     if isinstance(command, str):
-        command = command.split(" ")
+        command = shlex.split(command)
 
     python_libdir = sysconfig.get_config_var("LIBDIR")
-
     ld_library_path = get_env_var("LD_LIBRARY_PATH", "")
+
     if ld_library_path:
         python_libdir = ld_library_path + ":" + python_libdir
-    else:
-        python_libdir = python_libdir
 
-    env_vars = {
+    # Base environment variables setup
+    base_env = {
         "PATH": get_env_var("PATH", ""),
         "PYTHONPATH": get_env_var("PYTHONPATH", ""),
         "LD_LIBRARY_PATH": python_libdir,
-    } | env_vars
+    }
+
+    merged_env = base_env | env_vars
+    env_kv = [f"{k}={v}" for k, v in merged_env.items()]
+    return ["sudo", "-u", user, "env", *env_kv, *command]
 
-    env_kv = [f"{k}={v}" for k, v in env_vars.items()]
-    argv = ["sudo", "-u", user, "env", *env_kv, *command]
+
+def run_single_command_non_blocking(command: str | list, user: str, env_vars: dict):
+    """
+    Runs a single command non-blocking and returns the exit code after it finishes.
+    This is designed to be run within an Executor.
+    """
+    argv = prepare_command_argv(command, user, env_vars)
+    cmd_label = " ".join(argv[4:])  # For logging purposes
 
     try:
         proc = subprocess.Popen(argv)
         returncode = proc.wait()
+
         if returncode != 0:
-            logger.error(f"Command failed with exit code {returncode}")
-            sys.exit(returncode)
-    except subprocess.CalledProcessError as e:
-        logger.error(f"Command failed with exit code {e.returncode}")
-        logger.error(f"stderr: {e.stderr}")
-        sys.exit(1)
+            logger.error(f"Command '{cmd_label}' failed with exit code {returncode}")
+        return returncode
+    except Exception as e:
+        logger.error(f"Execution error for '{cmd_label}': {e}")
+        return 1
+
+
+def run_commands_in_parallel(
+    commands: list[str | list], user: str, env_vars: dict
+) -> tuple[int, int]:
+    """
+    Executes two commands concurrently using a ThreadPoolExecutor.
+    Returns a tuple of (return_code_cmd1, return_code_cmd2).
+    """
+    with concurrent.futures.ThreadPoolExecutor(max_workers=-1) as executor:
+        futures = []
+        for command in commands:
+            futures.append(
+                executor.submit(
+                    run_single_command_non_blocking, command, user, env_vars
+                )
+            )
+        results = [
+            future.result() for future in concurrent.futures.as_completed(futures)
+        ]
+        return results[0], results[1]
 
 
 def main():
@@ -137,7 +165,7 @@ def main():
             "DATATAILR_BATCH_ID": batch_id,
             "DATATAILR_BATCH_ENTRYPOINT": entrypoint,
         } | env
-        run_command_as_user("datatailr_run_batch", user, env)
+        run_single_command_non_blocking("datatailr_run_batch", user, env)
     elif job_type == "service":
         port = get_env_var("DATATAILR_SERVICE_PORT", 8080)
         entrypoint = get_env_var("DATATAILR_ENTRYPOINT")
@@ -145,13 +173,13 @@ def main():
             "DATATAILR_ENTRYPOINT": entrypoint,
             "DATATAILR_SERVICE_PORT": port,
         } | env
-        run_command_as_user("datatailr_run_service", user, env)
+        run_single_command_non_blocking("datatailr_run_service", user, env)
     elif job_type == "app":
         entrypoint = get_env_var("DATATAILR_ENTRYPOINT")
         env = {
             "DATATAILR_ENTRYPOINT": entrypoint,
         } | env
-        run_command_as_user("datatailr_run_app", user, env)
+        run_single_command_non_blocking("datatailr_run_app", user, env)
     elif job_type == "excel":
         host = get_env_var("DATATAILR_HOST", "")
         local = get_env_var("DATATAILR_LOCAL", "")
@@ -162,8 +190,12 @@ def main():
             "DATATAILR_HOST": host,
             "DATATAILR_LOCAL": local,
         } | env
-        run_command_as_user("datatailr_run_excel", user, env)
+        run_single_command_non_blocking("datatailr_run_excel", user, env)
     elif job_type == "workspace":
+        # Set a custom PS1 for the IDE terminal: 17:38|user@my-ide/~/dir/path:$
+        env["PS1"] = (
+            r"""\[\e[2m\]\A\[\e[0m\]|\[\e[38;5;40m\]\u\[\e[92m\]@${DATATAILR_JOB_NAME:-datatailr}\[\e[0m\]/\[\e[94;1m\]\w\[\e[0m\]\$"""
+        )
         ide_command = [
             "code-server",
             "--auth=none",
@@ -178,8 +210,7 @@ def main():
             "--NotebookApp.token=''",
             "--NotebookApp.password=''",
         ]
-        # TODO: run both commands in parallel
-        run_command_as_user(ide_command, user, env)
+        run_commands_in_parallel([ide_command, jupyter_command], user, env)
 
     else:
         raise ValueError(f"Unknown job type: {job_type}")

datatailr/sbin/datatailr_run_excel.py

@@ -35,6 +35,8 @@ def run():
             raise ValueError("Environment variable 'DATATAILR_HOST' is not set.")
         local_flag = ""
 
-    entrypoint = f'/opt/datatailr/bin/dt-excel.sh -n -H "{hostname}" {local_flag} -p 8080 -w 8000 {entrypoint}'
+    module_name = entrypoint.split(":", 1)[0]
+
+    entrypoint = f'/opt/datatailr/bin/dt-excel.sh -n -H "{hostname}" {local_flag} -p 8080 -w 8000 {module_name}'
     logger.info(f"Running entrypoint: {entrypoint}")
     subprocess.run(entrypoint, shell=True)

datatailr/scheduler/arguments_cache.py

@@ -28,6 +28,7 @@ from typing import Any, Dict, Optional
 
 from datatailr import is_dt_installed, Blob
 from datatailr.errors import DatatailrError
+from datatailr.wrapper import dt__Tag
 
 
 __BLOB_STORAGE__ = Blob()
@@ -48,11 +49,12 @@ class ArgumentsCache:
 
         :param use_persistent_cache: If True, use the persistent cache backend. Otherwise, use in-memory cache.
         """
+        self.__bucket_name__ = dt__Tag().get("blob_storage_prefix") + "batch"
         self.use_persistent_cache = use_persistent_cache
         if not self.use_persistent_cache:
             # Create a temp folder, for local caching
-            os.makedirs("/tmp/datatailr/batch/arguments", exist_ok=True)
-            os.makedirs("/tmp/datatailr/batch/results", exist_ok=True)
+            os.makedirs(f"/tmp/{self.__bucket_name__}/arguments", exist_ok=True)
+            os.makedirs(f"/tmp/{self.__bucket_name__}/results", exist_ok=True)
 
     def add_arguments(self, batch_id: str, arguments: Dict[str, Any]):
         """
@@ -62,7 +64,7 @@ class ArgumentsCache:
         :param job_name: Name of the job.
         :param arguments: Dictionary of arguments to store.
         """
-        path = f"/tmp/datatailr/batch/arguments/{batch_id}.pkl"
+        path = f"/tmp/{self.__bucket_name__}/arguments/{batch_id}.pkl"
         if self.use_persistent_cache:
             self._add_to_persistent_cache(path, arguments)
         else:
@@ -79,7 +81,7 @@ class ArgumentsCache:
         :param job_name: Name of the job.
         :return: Dictionary of arguments.
         """
-        path = f"/tmp/datatailr/batch/arguments/{batch_id}.pkl"
+        path = f"/tmp/{self.__bucket_name__}/arguments/{batch_id}.pkl"
         if self.use_persistent_cache and isinstance(job, str):
             try:
                 arg_keys = self._get_from_persistent_cache(path)
@@ -115,7 +117,7 @@ class ArgumentsCache:
         :param job: Name of the job.
         :param result: Result of the batch job.
         """
-        path = f"/tmp/datatailr/batch/results/{batch_run_id}/{job}.pkl"
+        path = f"/tmp/{self.__bucket_name__}/results/{batch_run_id}/{job}.pkl"
         if self.use_persistent_cache and isinstance(job, str):
             self._add_to_persistent_cache(path, result)
         else:
@@ -131,7 +133,7 @@ class ArgumentsCache:
         :param job: Name of the job.
         :return: Result of the batch job.
         """
-        path = f"/tmp/datatailr/batch/results/{batch_run_id}/{job}.pkl"
+        path = f"/tmp/{self.__bucket_name__}/results/{batch_run_id}/{job}.pkl"
         if self.use_persistent_cache and isinstance(job, str):
             return self._get_from_persistent_cache(path)
         else:

datatailr/scheduler/base.py

@@ -290,14 +290,30 @@ class Job:
         Returns a tuple of (branch: str, commit_hash: str).
         """
         path_to_repo = self.image.path_to_repo or "."
-        local_commit = run_shell_command(f"cd {path_to_repo} && git rev-parse HEAD")[0]
-        branch_name = run_shell_command(
-            f"cd {path_to_repo} && git rev-parse --abbrev-ref HEAD"
-        )[0]
+        try:
+            local_commit = run_shell_command(
+                f"cd {path_to_repo} && git rev-parse HEAD"
+            )[0]
+            branch_name = run_shell_command(
+                f"cd {path_to_repo} && git rev-parse --abbrev-ref HEAD"
+            )[0]
+            return_code = run_shell_command(
+                f"cd {path_to_repo} && git diff --exit-code"
+            )
+        except RuntimeError as e:
+            if "git: not found" in str(e):
+                logger.warning(
+                    "Git is not installed or not found in PATH. Repository validation is not possible."
+                )
+                branch_name, local_commit, return_code = "unknown", "unknown", None
+            else:
+                raise RepoValidationError(
+                    f"Error accessing git repository at {path_to_repo}: {e}"
+                ) from e
 
         if os.getenv("DATATAILR_ALLOW_UNSAFE_SCHEDULING", "false").lower() == "true":
             return branch_name, local_commit
-        return_code = run_shell_command(f"cd {path_to_repo} && git diff --exit-code")
+
         is_committed = return_code is not None and return_code[1] == 0
 
         if not is_committed:
@@ -337,6 +353,13 @@ class Job:
         """
         return {}
 
+    def __set_existing_id__(self, job_id: dict):
+        if isinstance(job_id, dict) and "Ids" in job_id and len(job_id["Ids"]) > 0:
+            job_id = job_id["Ids"][0]
+            if isinstance(job_id, str):
+                logger.debug(f"Setting existing job ID: {job_id}")
+                self.__id = job_id
+
     def __run_command__(self, command: str) -> Tuple[bool, str]:
         """
         Run a command in the context of the job.
@@ -350,18 +373,23 @@ class Job:
             temp_file_name = self.__prepare__()
 
             if command == "run":
-                __client__.run(f"file://{temp_file_name}", **self.get_schedule_args())
+                result = __client__.run(
+                    f"file://{temp_file_name}", **self.get_schedule_args()
+                )
             elif command == "save":
-                __client__.save(f"file://{temp_file_name}", **self.get_schedule_args())
+                result = __client__.save(
+                    f"file://{temp_file_name}", **self.get_schedule_args()
+                )
             elif command == "start":
-                __client__.start(self.name, environment=self.environment)
+                result = __client__.start(self.name, environment=self.environment)
             else:
                 raise ValueError(f"Unknown command: {command}")
             os.remove(temp_file_name)
         except Exception as e:
             logger.error(f"Error running command '{command}': {e}")
             return False, str(e)
-        return True, f"Job '{self.name}' {command}d successfully."
+        self.__set_existing_id__(result)
+        return True, result
 
     def save(self) -> Tuple[bool, str]:
         """

datatailr/scheduler/batch.py

@@ -501,8 +501,10 @@ class Batch(Job):
         __ARGUMENTS_CACHE__.add_arguments(self.id, args)
 
     def save(self) -> Tuple[bool, str]:
-        self.prepare_args()
-        return super().save()
+        status = super().save()
+        if status[0]:
+            self.prepare_args()
+        return status
 
     def run(self) -> Tuple[bool, str]:
         self.prepare_args()

datatailr/tag.py

@@ -0,0 +1,35 @@
+# *************************************************************************
+#
+#  Copyright (c) 2025 - Datatailr Inc.
+#  All Rights Reserved.
+#
+#  This file is part of Datatailr and subject to the terms and conditions
+#  defined in 'LICENSE.txt'. Unauthorized copying and/or distribution
+#  of this file, in parts or full, via any medium is strictly prohibited.
+# *************************************************************************
+
+from typing import Any, ClassVar
+
+
+class dt__Tag:
+    """
+    Tag management for local runs in the absence of the DataTailr platform.
+    All instances share the same tag store.
+    """
+
+    # shared across all instances
+    tags: ClassVar[dict[str, Any]] = {
+        "blob_storage_prefix": "local-no-dt-",
+    }
+
+    def ls(self) -> dict[str, Any]:
+        return self.__class__.tags
+
+    def get(self, name: str) -> Any:
+        return self.__class__.tags.get(name)
+
+    def set(self, name: str, value: Any) -> None:
+        self.__class__.tags[name] = value
+
+    def rm(self, name: str) -> None:
+        self.__class__.tags.pop(name, None)

datatailr/wrapper.py

@@ -14,6 +14,7 @@ import subprocess
 from typing import Union
 
 from datatailr.utils import is_dt_installed
+from datatailr.tag import dt__Tag as local_no_dt_Tag
 
 API_JSON_PATH: Union[str, None] = None
 
@@ -199,7 +200,7 @@ dt__Email = globals().get("dt__Email", mock_cli_tool)
 dt__Kv = globals().get("dt__Kv", mock_cli_tool)
 dt__Log = globals().get("dt__Log", mock_cli_tool)
 dt__Node = globals().get("dt__Node", mock_cli_tool)
-dt__Tag = globals().get("dt__Tag", mock_cli_tool)
+dt__Tag = globals().get("dt__Tag", local_no_dt_Tag)
 dt__Registry = globals().get("dt__Registry", mock_cli_tool)
 dt__Service = globals().get("dt__Service", mock_cli_tool)
 dt__Settings = globals().get("dt__Settings", mock_cli_tool)

{datatailr-0.1.67.dist-info → datatailr-0.1.68.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatailr
-Version: 0.1.67
+Version: 0.1.68
 Summary: Ready-to-Use Platform That Drives Business Insights
 Author-email: Datatailr <info@datatailr.com>
 License-Expression: MIT

{datatailr-0.1.67.dist-info → datatailr-0.1.68.dist-info}/RECORD

@@ -1,33 +1,34 @@
 datatailr/__init__.py,sha256=QTTG8X76BnlQwVx5N4ZQtSbLkgFipZ9NJGAbvtfuk_g,1051
-datatailr/acl.py,sha256=tlDy6VlHinSy5W1FbVxcNQNi7FliWUXy3ssIbzaPp28,4157
+datatailr/acl.py,sha256=7hBwF7TP_ADoDryYEFuXx2FCLavLmp3k_F0-sEXg26g,4173
 datatailr/blob.py,sha256=9lKZKm4eRKVE4_t2zy3hh6Z_Ov4iaTA3-tJFjnQKezg,3313
 datatailr/dt_json.py,sha256=3xmTqDBk68oPl2UW8UVOYPaBw4lAsVg6nDLwcen5nuo,2252
 datatailr/errors.py,sha256=p_e4ao3sFEfz1g4LvEDqw6bVzHJPJSINLjJ8H6_PqOo,751
 datatailr/group.py,sha256=AC0nCA44eEWZCJCq2klPqkFg_995mS3C_wu5uSFFLtU,4426
 datatailr/logging.py,sha256=msr0vzIaOtXmKjxsPpMw0Zl7jX9C_W0IuUwNjmqhhcA,4146
+datatailr/tag.py,sha256=YdTOBbbY5tYHJjiyaTgjsOprbwHLuc6jhLKDKXRlv7k,1081
 datatailr/user.py,sha256=tjIMbQ-tb2yFhgLBl_eTb47kKiy-RoM3KtYS3zmJiwc,6736
 datatailr/utils.py,sha256=0wEVD2UqBgfMRLO7DIL271ejSt-_0FU4ig1cSwnO5PI,2010
 datatailr/version.py,sha256=N9K8ZxlwFFSz8XSgbgaTWZY4k2J0JKfj698nZ_O2pIU,536
-datatailr/wrapper.py,sha256=2OEkMo6ls7sTddfS-ev-R1zKbYc194vDULQy7cm4Q8M,8025
+datatailr/wrapper.py,sha256=45RrMeYIFFWJAtOlQZRe1fT9daeq4vFlj6nIajbewEY,8080
 datatailr/build/__init__.py,sha256=_dA7b4L6wsaAFaSxUoYSJ1oaRqDHDMR20kqoCocSOss,487
 datatailr/build/image.py,sha256=YC8ML-l-sj6TcIBY-DCx_vaeI_7SmL9fPFhHnuxzRh0,5509
 datatailr/excel/__init__.py,sha256=IpXEPIFuu8IG3uQ8k7FDHqiHnChtyp-jgE8Plx2cWXQ,656
-datatailr/sbin/datatailr_run.py,sha256=8wZIeiqcbuZKC9K1HZnUo6iAbXcL0yQ7LG4YHAQ0XsM,7235
+datatailr/sbin/datatailr_run.py,sha256=iQpWHhlU-Y3gxyugTNcdh4Dkh8OHa-0KXEunGU9Kbxo,8478
 datatailr/sbin/datatailr_run_app.py,sha256=itF76XC2F4RK9s6bkoEppEiYwSLHK_5Jai3yvC-kFhY,1501
 datatailr/sbin/datatailr_run_batch.py,sha256=UWnp96j_G66R_Cape7Bb-rbK6UBLF7Y5_mTlWyGJAVQ,1818
-datatailr/sbin/datatailr_run_excel.py,sha256=MY2XH-T5bWABR23C_15eLhjaxggGRgjRhA-ByS6CtMc,1422
+datatailr/sbin/datatailr_run_excel.py,sha256=BLWmvxpKEE_8vJhs8E4VWq07FOBof5tlow-AkIEXtHw,1470
 datatailr/sbin/datatailr_run_service.py,sha256=DO9LGOpz3CVZOJJRHb4ac7AgY_mLbXHGadSyVCeIknc,1212
 datatailr/scheduler/__init__.py,sha256=qydHYVtEP6SUWd2CQ6FRdTdRWNz3SbYPJy4FK_wOvMk,1772
-datatailr/scheduler/arguments_cache.py,sha256=dl8IG9RTLSrJUk6g0_LYolkcuqW-LF52wdLPhs07LRI,6076
-datatailr/scheduler/base.py,sha256=0EN9YUiBgyYMzdsf4KFpNiQWTlCuV3NgCiauFHlYPbQ,15838
-datatailr/scheduler/batch.py,sha256=SBh9_G8WhQ8IkJw8NLjZ_HPbn5glIyDMQCrpaxd90Ko,17685
+datatailr/scheduler/arguments_cache.py,sha256=00OE0DhobYteBOnirjulO1ltgGBRamAdCO168O3_Zes,6236
+datatailr/scheduler/base.py,sha256=OWRblRCmKVe1stN43J35_g-1oKH4qteU4lrDRezyMV4,16829
+datatailr/scheduler/batch.py,sha256=CQCH1wHhW1qx09J7iQNQleErJ4n0nssAbd6u9YS6FMY,17735
 datatailr/scheduler/batch_decorator.py,sha256=LqL1bsupWLn-YEQUvFJYae7R3ogrL5-VodyiiScrkRw,5806
 datatailr/scheduler/constants.py,sha256=5WWTsfwZ_BA8gVDOTa2AQX9DJ0NzfaWgtY3vrODS2-8,606
 datatailr/scheduler/schedule.py,sha256=0XJJen2nL1xplRs0Xbjwgq3T-0bFCOrJzkSALdio998,3741
 datatailr/scheduler/utils.py,sha256=up6oR2iwe6G52LkvgfO394xchXgCYNjOMGRQW3e8PQk,1082
-datatailr-0.1.67.dist-info/licenses/LICENSE,sha256=ikKP4_O-UD_b8FuNdKmbzTb6odd0JX085ZW_FAPN3VI,1066
-datatailr-0.1.67.dist-info/METADATA,sha256=Y3m4aVMQtprItJcKYof3Fne2bACIrmAOp5at5JZxcWc,5146
-datatailr-0.1.67.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-datatailr-0.1.67.dist-info/entry_points.txt,sha256=YqXfk2At-olW4PUSRkqvy_O3Mbv7uTKCCPuAAiz3Qbg,312
-datatailr-0.1.67.dist-info/top_level.txt,sha256=75gntW0X_SKpqxLL6hAPipvpk28GAhJBvoyqN_HohWU,10
-datatailr-0.1.67.dist-info/RECORD,,
+datatailr-0.1.68.dist-info/licenses/LICENSE,sha256=ikKP4_O-UD_b8FuNdKmbzTb6odd0JX085ZW_FAPN3VI,1066
+datatailr-0.1.68.dist-info/METADATA,sha256=jSAMgwWq9plzAjbjmrKKGmYumhCa6dY7jqkyBBy-pRo,5146
+datatailr-0.1.68.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+datatailr-0.1.68.dist-info/entry_points.txt,sha256=YqXfk2At-olW4PUSRkqvy_O3Mbv7uTKCCPuAAiz3Qbg,312
+datatailr-0.1.68.dist-info/top_level.txt,sha256=75gntW0X_SKpqxLL6hAPipvpk28GAhJBvoyqN_HohWU,10
+datatailr-0.1.68.dist-info/RECORD,,