datatailr 0.1.20__tar.gz → 0.1.22__tar.gz

{datatailr-0.1.20/src/datatailr.egg-info → datatailr-0.1.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatailr
-Version: 0.1.20
+Version: 0.1.22
 Summary: Ready-to-Use Platform That Drives Business Insights
 Author-email: Datatailr <info@datatailr.com>
 License-Expression: MIT

{datatailr-0.1.20 → datatailr-0.1.22}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "datatailr"
-version = "0.1.20"
+version = "0.1.22"
 description = "Ready-to-Use Platform That Drives Business Insights"
 readme = "README.md"
 requires-python = ">=3.9"

datatailr-0.1.22/src/datatailr/excel.py

@@ -0,0 +1,19 @@
+try:
+    from dt.excel import Addin  # type: ignore
+except ImportError:
+
+    class DummyAddin:
+        def __init__(self, name: str, description: str) -> None:
+            self.name = name
+            self.description = description
+
+        def expose(self, **kwargs):
+            def decorator(func):
+                return func
+
+            return decorator
+
+        def run(self):
+            pass
+
+    Addin = DummyAddin

{datatailr-0.1.20 → datatailr-0.1.22}/src/datatailr/scheduler/arguments_cache.py

@@ -21,16 +21,18 @@ and the inner dictionaries contain the arguments.
 This module is for internal use of the datatailr package.
 """
 
-from datatailr.dt_json import json, decode_json
+from datatailr.dt_json import json
 import os
 import pickle
 from typing import Any, Dict, Optional
 
 from datatailr import is_dt_installed, Blob
 from datatailr.errors import DatatailrError
+from datatailr.logging import DatatailrLogger
 
 
 __BLOB_STORAGE__ = Blob()
+logger = DatatailrLogger(__name__).get_logger()
 
 
 class CacheNotFoundError(DatatailrError):
@@ -79,6 +81,9 @@ class ArgumentsCache:
         :param job_name: Name of the job.
         :return: Dictionary of arguments.
         """
+        logger.info(
+            f"Retrieving arguments for {batch_id=}, {job=}, {batch_run_id=}, {self.use_persistent_cache=}"
+        )
         path = f"/tmp/datatailr/batch/arguments/{batch_id}.pkl"
         if self.use_persistent_cache and isinstance(job, str):
             try:
@@ -101,12 +106,8 @@ class ArgumentsCache:
                     )
         if batch_run_id is None:
             return arg_keys[job]
-        arguments_mapping = decode_json(
-            os.getenv("DATATAILR_JOB_ARGUMENT_MAPPING", "{}")
-        )
-        arguments_mapping = {value: key for key, value in arguments_mapping.items()}
         args = {
-            arguments_mapping.get(name, name): self.get_result(batch_run_id, value)
+            name: self.get_result(batch_run_id, value)
             for name, value in arg_keys[job].items()
         }
         return args
@@ -120,6 +121,9 @@ class ArgumentsCache:
         :param result: Result of the batch job.
         """
         path = f"/tmp/datatailr/batch/results/{batch_run_id}_{job}.pkl"
+        logger.info(
+            f"Adding result for {batch_run_id=}, {job=}, {result=}, {self.use_persistent_cache=}"
+        )
         if self.use_persistent_cache and isinstance(job, str):
             self._add_to_persistent_cache(path, result)
         else:
@@ -156,6 +160,7 @@ class ArgumentsCache:
 
         """
         path = path.replace("/tmp/", "")
+        logger.info(f"Adding arguments to persistent cache for {path=}")
         __BLOB_STORAGE__.put_blob(path, json.dumps(blob))
 
     def _get_from_persistent_cache(self, path: str) -> Any:
@@ -165,5 +170,6 @@ class ArgumentsCache:
         :param path: Path in the Blob storage where the blob is stored.
         """
         path = path.replace("/tmp/", "")
+        logger.info(f"Retrieving arguments from persistent cache for {path=}")
         data = __BLOB_STORAGE__.get_blob(path)
         return json.loads(data)

{datatailr-0.1.20 → datatailr-0.1.22}/src/datatailr/scheduler/base.py

@@ -11,8 +11,7 @@
 from __future__ import annotations
 
 from datetime import datetime
-import importlib
-import inspect
+import importlib.util
 import json
 import os
 import tempfile
@@ -96,12 +95,17 @@ class EntryPoint:
 
         # Find the absolute path to the repository and then the relative path to the module.
         # This will be used in the creation of the code 'bundle' when building the image.
-        path_to_repo = run_shell_command("git rev-parse --show-toplevel")[0]
-        path_to_code = inspect.getfile(func)
-        package_root = path_to_code
+        module_spec = importlib.util.find_spec(func.__module__)
+        if module_spec is not None and module_spec.origin is not None:
+            package_root = module_spec.origin
+        else:
+            package_root = "."
         module_parts = self.module_name.split(".")
         for _ in module_parts:
             package_root = os.path.dirname(package_root)
+        path_to_repo = run_shell_command(
+            f"cd {package_root} && git rev-parse --show-toplevel"
+        )[0]
         path_to_module = os.path.relpath(package_root, path_to_repo)
         self.path_to_repo = path_to_repo
         self.path_to_module = path_to_module
@@ -123,7 +127,7 @@ class EntryPoint:
         return f"EntryPoint({self.function_name} from {self.module_name}, type={self.type})"
 
     def __str__(self):
-        return f"{self.module_name}.{self.function_name}"
+        return f"{self.module_name}:{self.function_name}"
 
 
 class Job:
@@ -277,20 +281,25 @@ class Job:
         2. Check if the local commit matches the remote HEAD (the repo is synced with the remote).
         Returns a tuple of (branch: str, commit_hash: str).
         """
-        local_commit = run_shell_command("git rev-parse HEAD")[0]
-        branch_name = run_shell_command("git rev-parse --abbrev-ref HEAD")[0]
+        path_to_repo = self.image.path_to_repo or "."
+        local_commit = run_shell_command(f"cd {path_to_repo} && git rev-parse HEAD")[0]
+        branch_name = run_shell_command(
+            f"cd {path_to_repo} && git rev-parse --abbrev-ref HEAD"
+        )[0]
 
         if os.getenv("DATATAILR_ALLOW_UNSAFE_SCHEDULING", "false").lower() == "true":
             return branch_name, local_commit
-        return_code = run_shell_command("git diff --exit-code")[1]
-        is_committed = return_code == 0
+        return_code = run_shell_command(f"cd {path_to_repo} && git diff --exit-code")
+        is_committed = return_code is not None and return_code[1] == 0
 
         if not is_committed:
             raise RepoValidationError(
                 "Please commit your changes before running the job."
             )
 
-        remote_commit = run_shell_command("git ls-remote origin HEAD")[0].split("\t")[0]
+        remote_commit = run_shell_command(
+            f"cd {path_to_repo} && git ls-remote origin HEAD"
+        )[0].split("\t")[0]
 
         if local_commit != remote_commit:
             raise RepoValidationError(

{datatailr-0.1.20 → datatailr-0.1.22}/src/datatailr/scheduler/batch.py

@@ -18,7 +18,6 @@ from typing import Any, Dict, List, Optional, Sequence, Set, Tuple, Union
 import uuid
 
 from datatailr import Image
-from datatailr.dt_json import encode_json
 from datatailr.errors import BatchJobError
 from datatailr.logging import DatatailrLogger
 from datatailr.scheduler.base import (
@@ -114,7 +113,7 @@ class BatchJob:
             isinstance(dep, int) for dep in self.dependencies
         ), "All dependencies must be integers representing job IDs."
         self.dag.add_job(self)
-        self.__argument_mapping = argument_mapping or {}
+        self.argument_mapping = argument_mapping or {}
 
     def __call__(self, *args, **kwds) -> BatchJob:
         """
@@ -200,6 +199,7 @@ class BatchJob:
         """
         return {
             "display_name": self.name,
+            "name": self.name,
             "child_number": self.__id,
             "entrypoint": str(self.entrypoint),
             "memory": self.resources.memory if self.resources else DEFAULT_TASK_MEMORY,
@@ -235,7 +235,7 @@ class BatchJob:
 
     def __add_dependency__(self, other):
         self.dependencies.add(other.__id)
-        arg_name = self.__argument_mapping.get(other.name, other.name)
+        arg_name = self.argument_mapping.get(other.name, other.name)
         if arg_name is not None:
             self.__args[arg_name] = other
 
@@ -282,7 +282,6 @@ class BatchJob:
                 "DATATAILR_BATCH_ID": str(self.dag.id),
                 "DATATAILR_JOB_ID": str(self.__id),
                 "DATATAILR_JOB_NAME": self.name,
-                "DATATAILR_JOB_ARGUMENT_MAPPING": encode_json(self.__argument_mapping),
             }
             self.entrypoint(env=env)
         else:
@@ -464,9 +463,23 @@ class Batch(Job):
         def arg_name(arg: Union[BatchJob, str]) -> str:
             return arg.name if isinstance(arg, BatchJob) else arg
 
+        def merged(dst: dict[str, str], src: dict[str, str]) -> dict[str, str]:
+            # copy so we don't mutate the original mapping
+            out = dict(dst)
+            seen_vals = set(out.values())
+            for k, v in src.items():
+                if v not in seen_vals:
+                    out[k] = v
+                    seen_vals.add(v)
+            return out
+
         args = {
-            j.name: {k: arg_name(v) for k, v in j.args.items()} for j in self.__jobs
+            j.name: merged(
+                j.argument_mapping, {k: arg_name(v) for k, v in j.args.items()}
+            )
+            for j in self.__jobs
         }
+
         __ARGUMENTS_CACHE__.add_arguments(self.id, args)
         if not self.__local_run and is_dt_installed():
             return super().run()

{datatailr-0.1.20 → datatailr-0.1.22}/src/datatailr/user.py

@@ -162,7 +162,7 @@ class User:
         last_name: str,
         email: str,
         password: str,
-        primary_group: int,
+        primary_group: str,
         is_system_user: bool = False,
     ) -> Optional["User"]:
         if is_system_user:

{datatailr-0.1.20 → datatailr-0.1.22/src/datatailr.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatailr
-Version: 0.1.20
+Version: 0.1.22
 Summary: Ready-to-Use Platform That Drives Business Insights
 Author-email: Datatailr <info@datatailr.com>
 License-Expression: MIT

{datatailr-0.1.20 → datatailr-0.1.22}/src/datatailr.egg-info/SOURCES.txt

@@ -7,6 +7,7 @@ src/datatailr/acl.py
 src/datatailr/blob.py
 src/datatailr/dt_json.py
 src/datatailr/errors.py
+src/datatailr/excel.py
 src/datatailr/group.py
 src/datatailr/logging.py
 src/datatailr/user.py

{datatailr-0.1.20 → datatailr-0.1.22}/src/sbin/datatailr_run.py

@@ -27,7 +27,6 @@
 #     DATATAILR_GID - the group ID of the group as it is defined in the system.
 #     DATATAILR_JOB_TYPE - the type of job to run. (batch\service\app\excel\IDE)
 # Job environment variables (not all are always relevant, depending on the job type):
-#     DATATAILR_JOB_ARGUMENT_MAPPING - a JSON string mapping job argument names to their
 #     DATATAILR_BATCH_RUN_ID - the unique identifier for the batch run.
 #     DATATAILR_BATCH_ID - the unique identifier for the batch.
 #     DATATAILR_JOB_ID - the unique identifier for the job.
@@ -37,7 +36,6 @@ import os
 import sys
 from typing import Tuple
 from datatailr.logging import DatatailrLogger
-from datatailr.dt_json import encode_json
 
 
 logger = DatatailrLogger(os.path.abspath(__file__)).get_logger()
@@ -83,7 +81,7 @@ def run_command_as_user(command: str, user: str, env_vars: dict):
     Run a command as a specific user with the given environment variables.
     """
     env_vars.update({"PATH": get_env_var("PATH")})
-    env_vars.update({"PYTHONPATH": get_env_var("PYTHONPATH")})
+    env_vars.update({"PYTHONPATH": get_env_var("PYTHONPATH", "")})
     env_vars_str = " ".join(f"{key}='{value}'" for key, value in env_vars.items())
     full_command = f"sudo -u {user} {env_vars_str} {command}"
     logger.debug(f"Running command: {full_command}")
@@ -103,16 +101,12 @@ def main():
     if job_type == "batch":
         run_id = get_env_var("DATATAILR_BATCH_RUN_ID")
         batch_id = get_env_var("DATATAILR_BATCH_ID")
-        job_argument_mapping = get_env_var(
-            "DATATAILR_JOB_ARGUMENT_MAPPING", encode_json({})
-        )
         entrypoint = get_env_var("DATATAILR_BATCH_ENTRYPOINT")
         env = {
             "DATATAILR_BATCH_RUN_ID": run_id,
             "DATATAILR_BATCH_ID": batch_id,
             "DATATAILR_JOB_ID": job_id,
             "DATATAILR_BATCH_ENTRYPOINT": entrypoint,
-            "DATATAILR_JOB_ARGUMENT_MAPPING": job_argument_mapping,
         }
         run_command_as_user("datatailr_run_batch", user, env)
     elif job_type == "service":