datatailr 0.1.34__tar.gz → 0.1.38__tar.gz

{datatailr-0.1.34/src/datatailr.egg-info → datatailr-0.1.38}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatailr
-Version: 0.1.34
+Version: 0.1.38
 Summary: Ready-to-Use Platform That Drives Business Insights
 Author-email: Datatailr <info@datatailr.com>
 License-Expression: MIT

{datatailr-0.1.34 → datatailr-0.1.38}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "datatailr"
-version = "0.1.34"
+version = "0.1.38"
 description = "Ready-to-Use Platform That Drives Business Insights"
 readme = "README.md"
 requires-python = ">=3.9"

{datatailr-0.1.34 → datatailr-0.1.38}/src/datatailr/blob.py

@@ -98,6 +98,10 @@ class Blob:
         # Since direct reading and writting of blobs is not implemented yet, we are using a temporary file.
         # This is a workaround to allow writing the blob content directly to the blob storage.
         with tempfile.NamedTemporaryFile(delete=True) as temp_file:
-            with open(temp_file.name, "w") as f:
-                f.write(blob)
+            if isinstance(blob, bytes):
+                with open(temp_file.name, "wb") as f:
+                    f.write(blob)
+            else:
+                with open(temp_file.name, "w") as f:
+                    f.write(blob)
             self.put_file(name, temp_file.name)

{datatailr-0.1.34 → datatailr-0.1.38}/src/datatailr/group.py

@@ -103,8 +103,8 @@ class Group:
 
     @staticmethod
     def add(name: str) -> Optional["Group"]:
-        __client__.add(name)
-        return Group.get(name)
+        new_group = __client__.add(name, json_enrichened=True)
+        return Group(new_group["name"]) if new_group else None
 
     @staticmethod
     def ls() -> list:

{datatailr-0.1.34 → datatailr-0.1.38}/src/datatailr/logging.py

@@ -10,7 +10,7 @@
 
 import logging
 import os
-from logging import StreamHandler
+import sys
 from logging.handlers import RotatingFileHandler
 from typing import Optional
 from datatailr import User
@@ -33,6 +33,28 @@ def get_log_level() -> int:
         return logging.INFO
 
 
+class MaxLevelFilter(logging.Filter):
+    """Allow only log records at or below a given level."""
+
+    def __init__(self, level):
+        super().__init__()
+        self.level = level
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        return record.levelno <= self.level
+
+
+class MinLevelFilter(logging.Filter):
+    """Allow only log records at or above a given level."""
+
+    def __init__(self, level):
+        super().__init__()
+        self.level = level
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        return record.levelno >= self.level
+
+
 tag = dt__Tag()
 node_name = tag.get("node_name") or "local"
 node_ip = tag.get("node_ip")
@@ -66,12 +88,21 @@ class DatatailrLogger:
         self.logger = logging.getLogger(name)
         self.logger.setLevel(log_level)
 
-        # Stream handler for stdout/stderr
-        stream_handler = StreamHandler()
-        stream_handler.setLevel(log_level)
-        stream_formatter = logging.Formatter(log_format)
-        stream_handler.setFormatter(stream_formatter)
-        self.logger.addHandler(stream_handler)
+        formatter = logging.Formatter(log_format)
+
+        # stdout handler (DEBUG/INFO only)
+        stdout_handler = logging.StreamHandler(sys.stdout)
+        stdout_handler.setLevel(logging.DEBUG)
+        stdout_handler.addFilter(MaxLevelFilter(logging.INFO))
+        stdout_handler.setFormatter(formatter)
+        self.logger.addHandler(stdout_handler)
+
+        # stderr handler (WARNING and above)
+        stderr_handler = logging.StreamHandler(sys.stderr)
+        stderr_handler.setLevel(logging.WARNING)
+        stderr_handler.addFilter(MinLevelFilter(logging.WARNING))
+        stderr_handler.setFormatter(formatter)
+        self.logger.addHandler(stderr_handler)
 
         # Optional file handler
         if log_file:
@@ -79,9 +110,9 @@ class DatatailrLogger:
                 log_file, maxBytes=10 * 1024 * 1024, backupCount=5
             )
             file_handler.setLevel(log_level)
-            file_formatter = logging.Formatter(log_format)
-            file_handler.setFormatter(file_formatter)
+            file_handler.setFormatter(formatter)
             self.logger.addHandler(file_handler)
+
         self.enable_opentelemetry()
 
     def get_logger(self):

{datatailr-0.1.34 → datatailr-0.1.38}/src/datatailr/scheduler/base.py

@@ -123,12 +123,10 @@ class EntryPoint:
             module = importlib.import_module(self.module_name)
             func = getattr(module, self.function_name)
             return func(*args, **kwargs)
-
-        elif self.type == JobType.SERVICE:
-            raise NotImplementedError("Service jobs are not yet implemented.")
-
-        elif self.type == JobType.APP:
-            raise NotImplementedError("App jobs are not yet implemented.")
+        else:
+            raise NotImplementedError(
+                f"EntryPoint of type '{self.type}' is not callable."
+            )
 
     def __repr__(self):
         return f"EntryPoint({self.function_name} from {self.module_name}, type={self.type})"

{datatailr-0.1.34 → datatailr-0.1.38}/src/datatailr/scheduler/batch.py

@@ -459,7 +459,7 @@ class Batch(Job):
             return args
         return {}
 
-    def run(self) -> Tuple[bool, str]:
+    def prepare_args(self) -> None:
         def arg_name(arg: Union[BatchJob, str]) -> str:
             return arg.name if isinstance(arg, BatchJob) else arg
 
@@ -481,6 +481,14 @@ class Batch(Job):
         }
 
         __ARGUMENTS_CACHE__.add_arguments(self.id, args)
+
+    def save(self) -> Tuple[bool, str]:
+        self.prepare_args()
+        return super().save()
+
+    def run(self) -> Tuple[bool, str]:
+        self.prepare_args()
+
         if not self.__local_run and is_dt_installed():
             return super().run()
         else:

{datatailr-0.1.34 → datatailr-0.1.38/src/datatailr.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatailr
-Version: 0.1.34
+Version: 0.1.38
 Summary: Ready-to-Use Platform That Drives Business Insights
 Author-email: Datatailr <info@datatailr.com>
 License-Expression: MIT

{datatailr-0.1.34 → datatailr-0.1.38}/src/sbin/datatailr_run.py

@@ -32,6 +32,7 @@
 #     DATATAILR_JOB_ID - the unique identifier for the job.
 
 
+import subprocess
 import os
 import sys
 from typing import Tuple
@@ -42,7 +43,7 @@ logger = DatatailrLogger(os.path.abspath(__file__)).get_logger()
 
 if not is_dt_installed():
     logger.error("Datatailr is not installed.")
-    sys.exit(1)
+    # sys.exit(1) # TODO: Uncomment after testing
 
 
 def get_env_var(name: str, default: str | None = None) -> str:
@@ -80,18 +81,32 @@ def create_user_and_group() -> Tuple[str, str]:
     return user, group
 
 
-def run_command_as_user(command: str, user: str, env_vars: dict):
+def run_command_as_user(command: str | list, user: str, env_vars: dict):
     """
     Run a command as a specific user with the given environment variables.
     """
-    env_vars.update({"PATH": get_env_var("PATH")})
-    env_vars.update({"PYTHONPATH": get_env_var("PYTHONPATH", "")})
-    env_vars_str = " ".join(f"{key}='{value}'" for key, value in env_vars.items())
-    full_command = f"sudo -u {user} {env_vars_str} {command}"
-    logger.debug(f"Running command: {full_command}")
-    status = os.system(full_command)
-    if status != 0:
-        logger.error(f"Command failed with status {status / 256:g}")
+    if isinstance(command, str):
+        command = command.split(" ")
+    env_vars = {
+        "PATH": get_env_var("PATH", ""),
+        "PYTHONPATH": get_env_var("PYTHONPATH", ""),
+    } | env_vars
+
+    env_kv = [f"{k}={v}" for k, v in env_vars.items()]
+    argv = ["sudo", "-u", user, "env", *env_kv, "bash", "-lc", *command]
+
+    try:
+        result = subprocess.run(
+            argv,
+            check=True,  # raises if returncode != 0
+            capture_output=True,
+            text=True,
+        )
+        logger.info(f"stdout: {result.stdout}")
+        logger.debug(f"stderr: {result.stderr}")
+    except subprocess.CalledProcessError as e:
+        logger.error(f"Command failed with exit code {e.returncode}")
+        logger.error(f"stderr: {e.stderr}")
         sys.exit(1)
 
 
@@ -99,8 +114,11 @@ def main():
     user, _ = create_user_and_group()
     job_type = get_env_var("DATATAILR_JOB_TYPE")
 
-    job_name = get_env_var("DATATAILR_JOB_NAME")
-    job_id = get_env_var("DATATAILR_JOB_ID")
+    env = {
+        "DATATAILR_JOB_TYPE": job_type,
+        "DATATAILR_JOB_NAME": get_env_var("DATATAILR_JOB_NAME"),
+        "DATATAILR_JOB_ID": get_env_var("DATATAILR_JOB_ID"),
+    }
 
     if job_type == "batch":
         run_id = get_env_var("DATATAILR_BATCH_RUN_ID")
@@ -109,40 +127,49 @@ def main():
         env = {
             "DATATAILR_BATCH_RUN_ID": run_id,
             "DATATAILR_BATCH_ID": batch_id,
-            "DATATAILR_JOB_ID": job_id,
             "DATATAILR_BATCH_ENTRYPOINT": entrypoint,
-        }
+        } | env
         run_command_as_user("datatailr_run_batch", user, env)
     elif job_type == "service":
         port = get_env_var("DATATAILR_SERVICE_PORT")
         entrypoint = get_env_var("DATATAILR_ENTRYPOINT")
         env = {
-            "DATATAILR_JOB_NAME": job_name,
-            "DATATAILR_JOB_ID": job_id,
             "DATATAILR_ENTRYPOINT": entrypoint,
             "DATATAILR_SERVICE_PORT": port,
-        }
+        } | env
         run_command_as_user("datatailr_run_service", user, env)
     elif job_type == "app":
         entrypoint = get_env_var("DATATAILR_ENTRYPOINT")
         env = {
-            "DATATAILR_JOB_NAME": job_name,
-            "DATATAILR_JOB_ID": job_id,
             "DATATAILR_ENTRYPOINT": entrypoint,
-        }
+        } | env
         run_command_as_user("datatailr_run_app", user, env)
     elif job_type == "excel":
-        host = get_env_var("DATATAILR_HOST")
+        host = get_env_var("DATATAILR_HOST", "")
         entrypoint = get_env_var("DATATAILR_ENTRYPOINT")
         env = {
-            "DATATAILR_JOB_NAME": job_name,
-            "DATATAILR_JOB_ID": job_id,
             "DATATAILR_ENTRYPOINT": entrypoint,
             "DATATAILR_HOST": host,
-        }
+        } | env
         run_command_as_user("datatailr_run_excel", user, env)
-    elif job_type == "IDE":
-        pass
+    elif job_type == "ide":
+        command = [
+            "code-server",
+            "--auth=none",
+            "--bind-addr=0.0.0.0:8080",
+            f'--app-name="Datatailr IDE {get_env_var("DATATAILR_USER")}"',
+        ]
+        run_command_as_user(command, user, env)
+    elif job_type == "jupyter":
+        command = [
+            "jupyter-lab",
+            "--ip='*'",
+            "--port=8080",
+            "--no-browser",
+            "--NotebookApp.token=''",
+            "--NotebookApp.password=''",
+        ]
+        run_command_as_user(command, user, env)
     else:
         raise ValueError(f"Unknown job type: {job_type}")