datatailr 0.1.21__py3-none-any.whl → 0.1.22__py3-none-any.whl

datatailr/excel.py

@@ -1,5 +1,5 @@
 try:
-    from dt.excel import Addin
+    from dt.excel import Addin  # type: ignore
 except ImportError:
 
     class DummyAddin:

datatailr/sbin/datatailr_run.py

@@ -27,7 +27,6 @@
 #     DATATAILR_GID - the group ID of the group as it is defined in the system.
 #     DATATAILR_JOB_TYPE - the type of job to run. (batch\service\app\excel\IDE)
 # Job environment variables (not all are always relevant, depending on the job type):
-#     DATATAILR_JOB_ARGUMENT_MAPPING - a JSON string mapping job argument names to their
 #     DATATAILR_BATCH_RUN_ID - the unique identifier for the batch run.
 #     DATATAILR_BATCH_ID - the unique identifier for the batch.
 #     DATATAILR_JOB_ID - the unique identifier for the job.
@@ -37,7 +36,6 @@ import os
 import sys
 from typing import Tuple
 from datatailr.logging import DatatailrLogger
-from datatailr.dt_json import encode_json
 
 
 logger = DatatailrLogger(os.path.abspath(__file__)).get_logger()
@@ -83,7 +81,7 @@ def run_command_as_user(command: str, user: str, env_vars: dict):
     Run a command as a specific user with the given environment variables.
     """
     env_vars.update({"PATH": get_env_var("PATH")})
-    env_vars.update({"PYTHONPATH": get_env_var("PYTHONPATH")})
+    env_vars.update({"PYTHONPATH": get_env_var("PYTHONPATH", "")})
     env_vars_str = " ".join(f"{key}='{value}'" for key, value in env_vars.items())
     full_command = f"sudo -u {user} {env_vars_str} {command}"
     logger.debug(f"Running command: {full_command}")
@@ -103,16 +101,12 @@ def main():
     if job_type == "batch":
         run_id = get_env_var("DATATAILR_BATCH_RUN_ID")
         batch_id = get_env_var("DATATAILR_BATCH_ID")
-        job_argument_mapping = get_env_var(
-            "DATATAILR_JOB_ARGUMENT_MAPPING", encode_json({})
-        )
         entrypoint = get_env_var("DATATAILR_BATCH_ENTRYPOINT")
         env = {
             "DATATAILR_BATCH_RUN_ID": run_id,
             "DATATAILR_BATCH_ID": batch_id,
             "DATATAILR_JOB_ID": job_id,
             "DATATAILR_BATCH_ENTRYPOINT": entrypoint,
-            "DATATAILR_JOB_ARGUMENT_MAPPING": job_argument_mapping,
         }
         run_command_as_user("datatailr_run_batch", user, env)
     elif job_type == "service":

datatailr/scheduler/arguments_cache.py

@@ -21,16 +21,18 @@ and the inner dictionaries contain the arguments.
 This module is for internal use of the datatailr package.
 """
 
-from datatailr.dt_json import json, decode_json
+from datatailr.dt_json import json
 import os
 import pickle
 from typing import Any, Dict, Optional
 
 from datatailr import is_dt_installed, Blob
 from datatailr.errors import DatatailrError
+from datatailr.logging import DatatailrLogger
 
 
 __BLOB_STORAGE__ = Blob()
+logger = DatatailrLogger(__name__).get_logger()
 
 
 class CacheNotFoundError(DatatailrError):
@@ -79,6 +81,9 @@ class ArgumentsCache:
         :param job_name: Name of the job.
         :return: Dictionary of arguments.
         """
+        logger.info(
+            f"Retrieving arguments for {batch_id=}, {job=}, {batch_run_id=}, {self.use_persistent_cache=}"
+        )
         path = f"/tmp/datatailr/batch/arguments/{batch_id}.pkl"
         if self.use_persistent_cache and isinstance(job, str):
             try:
@@ -101,12 +106,8 @@ class ArgumentsCache:
                     )
         if batch_run_id is None:
             return arg_keys[job]
-        arguments_mapping = decode_json(
-            os.getenv("DATATAILR_JOB_ARGUMENT_MAPPING", "{}")
-        )
-        arguments_mapping = {value: key for key, value in arguments_mapping.items()}
         args = {
-            arguments_mapping.get(name, name): self.get_result(batch_run_id, value)
+            name: self.get_result(batch_run_id, value)
             for name, value in arg_keys[job].items()
         }
         return args
@@ -120,6 +121,9 @@ class ArgumentsCache:
         :param result: Result of the batch job.
         """
         path = f"/tmp/datatailr/batch/results/{batch_run_id}_{job}.pkl"
+        logger.info(
+            f"Adding result for {batch_run_id=}, {job=}, {result=}, {self.use_persistent_cache=}"
+        )
         if self.use_persistent_cache and isinstance(job, str):
             self._add_to_persistent_cache(path, result)
         else:
@@ -156,6 +160,7 @@ class ArgumentsCache:
 
         """
         path = path.replace("/tmp/", "")
+        logger.info(f"Adding arguments to persistent cache for {path=}")
         __BLOB_STORAGE__.put_blob(path, json.dumps(blob))
 
     def _get_from_persistent_cache(self, path: str) -> Any:
@@ -165,5 +170,6 @@ class ArgumentsCache:
         :param path: Path in the Blob storage where the blob is stored.
         """
         path = path.replace("/tmp/", "")
+        logger.info(f"Retrieving arguments from persistent cache for {path=}")
         data = __BLOB_STORAGE__.get_blob(path)
         return json.loads(data)

datatailr/scheduler/base.py

@@ -11,8 +11,7 @@
 from __future__ import annotations
 
 from datetime import datetime
-import importlib
-import inspect
+import importlib.util
 import json
 import os
 import tempfile
@@ -96,12 +95,17 @@ class EntryPoint:
 
         # Find the absolute path to the repository and then the relative path to the module.
         # This will be used in the creation of the code 'bundle' when building the image.
-        path_to_repo = run_shell_command("git rev-parse --show-toplevel")[0]
-        path_to_code = inspect.getfile(func)
-        package_root = path_to_code
+        module_spec = importlib.util.find_spec(func.__module__)
+        if module_spec is not None and module_spec.origin is not None:
+            package_root = module_spec.origin
+        else:
+            package_root = "."
         module_parts = self.module_name.split(".")
         for _ in module_parts:
             package_root = os.path.dirname(package_root)
+        path_to_repo = run_shell_command(
+            f"cd {package_root} && git rev-parse --show-toplevel"
+        )[0]
         path_to_module = os.path.relpath(package_root, path_to_repo)
         self.path_to_repo = path_to_repo
         self.path_to_module = path_to_module
@@ -123,7 +127,7 @@ class EntryPoint:
         return f"EntryPoint({self.function_name} from {self.module_name}, type={self.type})"
 
     def __str__(self):
-        return f"{self.module_name}.{self.function_name}"
+        return f"{self.module_name}:{self.function_name}"
 
 
 class Job:
@@ -277,20 +281,25 @@ class Job:
         2. Check if the local commit matches the remote HEAD (the repo is synced with the remote).
         Returns a tuple of (branch: str, commit_hash: str).
         """
-        local_commit = run_shell_command("git rev-parse HEAD")[0]
-        branch_name = run_shell_command("git rev-parse --abbrev-ref HEAD")[0]
+        path_to_repo = self.image.path_to_repo or "."
+        local_commit = run_shell_command(f"cd {path_to_repo} && git rev-parse HEAD")[0]
+        branch_name = run_shell_command(
+            f"cd {path_to_repo} && git rev-parse --abbrev-ref HEAD"
+        )[0]
 
         if os.getenv("DATATAILR_ALLOW_UNSAFE_SCHEDULING", "false").lower() == "true":
             return branch_name, local_commit
-        return_code = run_shell_command("git diff --exit-code")[1]
-        is_committed = return_code == 0
+        return_code = run_shell_command(f"cd {path_to_repo} && git diff --exit-code")
+        is_committed = return_code is not None and return_code[1] == 0
 
         if not is_committed:
             raise RepoValidationError(
                 "Please commit your changes before running the job."
             )
 
-        remote_commit = run_shell_command("git ls-remote origin HEAD")[0].split("\t")[0]
+        remote_commit = run_shell_command(
+            f"cd {path_to_repo} && git ls-remote origin HEAD"
+        )[0].split("\t")[0]
 
         if local_commit != remote_commit:
             raise RepoValidationError(

datatailr/scheduler/batch.py

@@ -18,7 +18,6 @@ from typing import Any, Dict, List, Optional, Sequence, Set, Tuple, Union
 import uuid
 
 from datatailr import Image
-from datatailr.dt_json import encode_json
 from datatailr.errors import BatchJobError
 from datatailr.logging import DatatailrLogger
 from datatailr.scheduler.base import (
@@ -114,7 +113,7 @@ class BatchJob:
             isinstance(dep, int) for dep in self.dependencies
         ), "All dependencies must be integers representing job IDs."
         self.dag.add_job(self)
-        self.__argument_mapping = argument_mapping or {}
+        self.argument_mapping = argument_mapping or {}
 
     def __call__(self, *args, **kwds) -> BatchJob:
         """
@@ -200,6 +199,7 @@ class BatchJob:
         """
         return {
             "display_name": self.name,
+            "name": self.name,
             "child_number": self.__id,
             "entrypoint": str(self.entrypoint),
             "memory": self.resources.memory if self.resources else DEFAULT_TASK_MEMORY,
@@ -235,7 +235,7 @@ class BatchJob:
 
     def __add_dependency__(self, other):
         self.dependencies.add(other.__id)
-        arg_name = self.__argument_mapping.get(other.name, other.name)
+        arg_name = self.argument_mapping.get(other.name, other.name)
         if arg_name is not None:
             self.__args[arg_name] = other
 
@@ -282,7 +282,6 @@ class BatchJob:
                 "DATATAILR_BATCH_ID": str(self.dag.id),
                 "DATATAILR_JOB_ID": str(self.__id),
                 "DATATAILR_JOB_NAME": self.name,
-                "DATATAILR_JOB_ARGUMENT_MAPPING": encode_json(self.__argument_mapping),
             }
             self.entrypoint(env=env)
         else:
@@ -464,9 +463,23 @@ class Batch(Job):
         def arg_name(arg: Union[BatchJob, str]) -> str:
             return arg.name if isinstance(arg, BatchJob) else arg
 
+        def merged(dst: dict[str, str], src: dict[str, str]) -> dict[str, str]:
+            # copy so we don't mutate the original mapping
+            out = dict(dst)
+            seen_vals = set(out.values())
+            for k, v in src.items():
+                if v not in seen_vals:
+                    out[k] = v
+                    seen_vals.add(v)
+            return out
+
         args = {
-            j.name: {k: arg_name(v) for k, v in j.args.items()} for j in self.__jobs
+            j.name: merged(
+                j.argument_mapping, {k: arg_name(v) for k, v in j.args.items()}
+            )
+            for j in self.__jobs
         }
+
         __ARGUMENTS_CACHE__.add_arguments(self.id, args)
         if not self.__local_run and is_dt_installed():
             return super().run()

datatailr/user.py

@@ -162,7 +162,7 @@ class User:
         last_name: str,
         email: str,
         password: str,
-        primary_group: int,
+        primary_group: str,
         is_system_user: bool = False,
     ) -> Optional["User"]:
         if is_system_user:

{datatailr-0.1.21.dist-info → datatailr-0.1.22.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatailr
-Version: 0.1.21
+Version: 0.1.22
 Summary: Ready-to-Use Platform That Drives Business Insights
 Author-email: Datatailr <info@datatailr.com>
 License-Expression: MIT

{datatailr-0.1.21.dist-info → datatailr-0.1.22.dist-info}/RECORD

@@ -3,31 +3,31 @@ datatailr/acl.py,sha256=tlDy6VlHinSy5W1FbVxcNQNi7FliWUXy3ssIbzaPp28,4157
 datatailr/blob.py,sha256=xkXT6RZcMww4YfLVjOyqvvPxWc-Ku6fTJ_PeCXyBys4,3159
 datatailr/dt_json.py,sha256=3xmTqDBk68oPl2UW8UVOYPaBw4lAsVg6nDLwcen5nuo,2252
 datatailr/errors.py,sha256=p_e4ao3sFEfz1g4LvEDqw6bVzHJPJSINLjJ8H6_PqOo,751
-datatailr/excel.py,sha256=YqIkaM5ap6uKj92yH6YGoB6I1oN10S-1zl5Uf9_xXx4,409
+datatailr/excel.py,sha256=7_13VFHrjHlIAEeTaGQIlq4ycJvzRo-TvkP_KRBELGs,425
 datatailr/group.py,sha256=ExsrkqUooAfFLWKvnkp1ZxisSJD1yCp9TKqoCXDCwhs,4360
 datatailr/logging.py,sha256=4Rsx3wf2tAr1334E2goBjhC877RwbUNaFgBlh902vU4,3270
-datatailr/user.py,sha256=tY6TcwOvq1SW6Uqot6eYR3SBAihPwWKs2qDfHDvF4xw,6624
+datatailr/user.py,sha256=THAHiHgcr6MGoF2EuFEUfPh-zRT4E_WhG6eoVNBNeKo,6624
 datatailr/utils.py,sha256=mqnnERMyHNAuAgFY4Ry4O4yW0ZjCRtJbjfI5fXVqt2s,1524
 datatailr/version.py,sha256=N9K8ZxlwFFSz8XSgbgaTWZY4k2J0JKfj698nZ_O2pIU,536
 datatailr/wrapper.py,sha256=K9ZD76cWey_ikA6C5sKejwRaYBDln4QMg-RcoRGiuFc,7991
 datatailr/build/__init__.py,sha256=_dA7b4L6wsaAFaSxUoYSJ1oaRqDHDMR20kqoCocSOss,487
 datatailr/build/image.py,sha256=YC8ML-l-sj6TcIBY-DCx_vaeI_7SmL9fPFhHnuxzRh0,5509
-datatailr/sbin/datatailr_run.py,sha256=_P2q0-i3JS0sPt49eX_EhjPxCrJyfi9-FGoYcqND4Zs,6307
+datatailr/sbin/datatailr_run.py,sha256=l3xoX_Ex-U7vXCiH343D3SQnqP6BY-xIx5qB6iES5ns,5996
 datatailr/sbin/datatailr_run_app.py,sha256=AOkutzv4DeKfWZs-ZBciAMKnK4A05SfkVf1ZJnSSFwA,1231
 datatailr/sbin/datatailr_run_batch.py,sha256=UWnp96j_G66R_Cape7Bb-rbK6UBLF7Y5_mTlWyGJAVQ,1818
 datatailr/sbin/datatailr_run_excel.py,sha256=Gr_QZgqJrwgRVD9_o4v-2tbvU-QMvNHL7xUvFGhftFc,1163
 datatailr/sbin/datatailr_run_service.py,sha256=R8eNLN2SGnMtyfLy3vq9isUHr3dRzeBqESTquNK9Iho,1156
 datatailr/scheduler/__init__.py,sha256=qydHYVtEP6SUWd2CQ6FRdTdRWNz3SbYPJy4FK_wOvMk,1772
-datatailr/scheduler/arguments_cache.py,sha256=CydYR9o2pqfa4KsPTA1mJSBN-0YF47Q6AmODm4zAJQ4,6254
-datatailr/scheduler/base.py,sha256=XZexeOGWHOSI0babRB8XCvAnL3fAQSXYWRfj7srMAkk,14240
-datatailr/scheduler/batch.py,sha256=77iUufBqRBau9Ku4IivLOKqh-lknclEak8jg2YhsX3c,16437
+datatailr/scheduler/arguments_cache.py,sha256=ct9CiUF3IDB5S7dsQWFarc8yQXa6X8xvRrKjJ1zYKhM,6527
+datatailr/scheduler/base.py,sha256=4K39tQPpYnrhf2PmFWlcp0xkH2qEwhkfflt8qpWhxnQ,14633
+datatailr/scheduler/batch.py,sha256=JBSiFNwv2X2BkBocz7ddVmwND6MFds978f0Ok5f8aUc,16782
 datatailr/scheduler/batch_decorator.py,sha256=LqL1bsupWLn-YEQUvFJYae7R3ogrL5-VodyiiScrkRw,5806
 datatailr/scheduler/constants.py,sha256=5WWTsfwZ_BA8gVDOTa2AQX9DJ0NzfaWgtY3vrODS2-8,606
 datatailr/scheduler/schedule.py,sha256=0XJJen2nL1xplRs0Xbjwgq3T-0bFCOrJzkSALdio998,3741
 datatailr/scheduler/utils.py,sha256=up6oR2iwe6G52LkvgfO394xchXgCYNjOMGRQW3e8PQk,1082
-datatailr-0.1.21.dist-info/licenses/LICENSE,sha256=ikKP4_O-UD_b8FuNdKmbzTb6odd0JX085ZW_FAPN3VI,1066
-datatailr-0.1.21.dist-info/METADATA,sha256=b1iOwgH34LZmK3e-D1gBtAhLjCmuvlW_Ip-D4LYWN6I,5146
-datatailr-0.1.21.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-datatailr-0.1.21.dist-info/entry_points.txt,sha256=YqXfk2At-olW4PUSRkqvy_O3Mbv7uTKCCPuAAiz3Qbg,312
-datatailr-0.1.21.dist-info/top_level.txt,sha256=75gntW0X_SKpqxLL6hAPipvpk28GAhJBvoyqN_HohWU,10
-datatailr-0.1.21.dist-info/RECORD,,
+datatailr-0.1.22.dist-info/licenses/LICENSE,sha256=ikKP4_O-UD_b8FuNdKmbzTb6odd0JX085ZW_FAPN3VI,1066
+datatailr-0.1.22.dist-info/METADATA,sha256=KplXEwrUeqWjlRjkyN3NenplmPDlWPNzUsmCxbkD83Q,5146
+datatailr-0.1.22.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+datatailr-0.1.22.dist-info/entry_points.txt,sha256=YqXfk2At-olW4PUSRkqvy_O3Mbv7uTKCCPuAAiz3Qbg,312
+datatailr-0.1.22.dist-info/top_level.txt,sha256=75gntW0X_SKpqxLL6hAPipvpk28GAhJBvoyqN_HohWU,10
+datatailr-0.1.22.dist-info/RECORD,,