datatailr 0.1.68__tar.gz → 0.1.70__tar.gz

{datatailr-0.1.68/src/datatailr.egg-info → datatailr-0.1.70}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatailr
-Version: 0.1.68
+Version: 0.1.70
 Summary: Ready-to-Use Platform That Drives Business Insights
 Author-email: Datatailr <info@datatailr.com>
 License-Expression: MIT

{datatailr-0.1.68 → datatailr-0.1.70}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "datatailr"
-version = "0.1.68"
+version = "0.1.70"
 description = "Ready-to-Use Platform That Drives Business Insights"
 readme = "README.md"
 requires-python = ">=3.9"

{datatailr-0.1.68 → datatailr-0.1.70}/src/datatailr/scheduler/arguments_cache.py

@@ -49,7 +49,10 @@ class ArgumentsCache:
 
         :param use_persistent_cache: If True, use the persistent cache backend. Otherwise, use in-memory cache.
         """
-        self.__bucket_name__ = dt__Tag().get("blob_storage_prefix") + "batch"
+        try:
+            self.__bucket_name__ = dt__Tag().get("blob_storage_prefix") + "batch"
+        except Exception:
+            self.__bucket_name__ = "local-batch"
         self.use_persistent_cache = use_persistent_cache
         if not self.use_persistent_cache:
             # Create a temp folder, for local caching

{datatailr-0.1.68 → datatailr-0.1.70/src/datatailr.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatailr
-Version: 0.1.68
+Version: 0.1.70
 Summary: Ready-to-Use Platform That Drives Business Insights
 Author-email: Datatailr <info@datatailr.com>
 License-Expression: MIT

{datatailr-0.1.68 → datatailr-0.1.70}/src/sbin/datatailr_run.py

@@ -37,7 +37,7 @@ import subprocess
 import os
 import shlex
 import sysconfig
-from typing import Tuple
+from typing import Optional, Tuple
 from datatailr.logging import DatatailrLogger
 from datatailr.utils import is_dt_installed
 
@@ -105,7 +105,12 @@ def prepare_command_argv(command: str | list, user: str, env_vars: dict) -> list
     return ["sudo", "-u", user, "env", *env_kv, *command]
 
 
-def run_single_command_non_blocking(command: str | list, user: str, env_vars: dict):
+def run_single_command_non_blocking(
+    command: str | list,
+    user: str,
+    env_vars: dict,
+    log_stream_name: Optional[str | None] = None,
+) -> int:
     """
     Runs a single command non-blocking and returns the exit code after it finishes.
     This is designed to be run within an Executor.
@@ -114,7 +119,16 @@ def run_single_command_non_blocking(command: str | list, user: str, env_vars: di
     cmd_label = " ".join(argv[4:])  # For logging purposes
 
     try:
-        proc = subprocess.Popen(argv)
+        if log_stream_name:
+            stdout_file_path = f"/opt/datatailr/var/log/{log_stream_name}.log"
+            stderr_file_path = f"/opt/datatailr/var/log/{log_stream_name}_error.log"
+            with (
+                open(stdout_file_path, "ab", buffering=0) as stdout_file,
+                open(stderr_file_path, "ab", buffering=0) as stderr_file,
+            ):
+                proc = subprocess.Popen(argv, stdout=stdout_file, stderr=stderr_file)
+        else:
+            proc = subprocess.Popen(argv)
         returncode = proc.wait()
 
         if returncode != 0:
@@ -126,18 +140,27 @@ def run_single_command_non_blocking(command: str | list, user: str, env_vars: di
 
 
 def run_commands_in_parallel(
-    commands: list[str | list], user: str, env_vars: dict
+    commands: list[str | list],
+    user: str,
+    env_vars: dict,
+    log_stream_names: Optional[list[str | None]] = None,
 ) -> tuple[int, int]:
     """
     Executes two commands concurrently using a ThreadPoolExecutor.
     Returns a tuple of (return_code_cmd1, return_code_cmd2).
     """
-    with concurrent.futures.ThreadPoolExecutor(max_workers=-1) as executor:
+    with concurrent.futures.ThreadPoolExecutor(max_workers=len(commands)) as executor:
         futures = []
-        for command in commands:
+        for command, log_stream_name in zip(
+            commands, log_stream_names or [None] * len(commands)
+        ):
             futures.append(
                 executor.submit(
-                    run_single_command_non_blocking, command, user, env_vars
+                    run_single_command_non_blocking,
+                    command,
+                    user,
+                    env_vars,
+                    log_stream_name,
                 )
             )
         results = [
@@ -196,6 +219,7 @@ def main():
         env["PS1"] = (
             r"""\[\e[2m\]\A\[\e[0m\]|\[\e[38;5;40m\]\u\[\e[92m\]@${DATATAILR_JOB_NAME:-datatailr}\[\e[0m\]/\[\e[94;1m\]\w\[\e[0m\]\$"""
         )
+        os.makedirs("/opt/datatailr/var/log", exist_ok=True)
         ide_command = [
             "code-server",
             "--auth=none",
@@ -210,7 +234,9 @@ def main():
             "--NotebookApp.token=''",
             "--NotebookApp.password=''",
         ]
-        run_commands_in_parallel([ide_command, jupyter_command], user, env)
+        run_commands_in_parallel(
+            [ide_command, jupyter_command], user, env, ["code-server", "jupyter"]
+        )
 
     else:
         raise ValueError(f"Unknown job type: {job_type}")