datatailr 0.1.70__py3-none-any.whl → 0.1.72__py3-none-any.whl

datatailr/blob.py

@@ -10,12 +10,14 @@
 
 from __future__ import annotations
 
+import os
 import tempfile
 
 from datatailr.wrapper import dt__Blob
 
 # Datatailr Blob API Client
 __client__ = dt__Blob()
+__user__ = os.getenv("USER", "root")
 
 
 class Blob:
@@ -81,8 +83,13 @@ class Blob:
         """
         # Since direct reading and writting of blobs is not implemented yet, we are using a temporary file.
         # This is a workaround to allow reading the blob content directly from the blob storage.
-
-        with tempfile.NamedTemporaryFile(delete=True) as temp_file:
+        temp_dir = f"/home/{__user__}/tmp"
+        if not os.path.exists(temp_dir):
+            temp_dir = "/tmp"
+        else:
+            temp_dir += "/.dt"
+            os.makedirs(temp_dir, exist_ok=True)
+        with tempfile.NamedTemporaryFile(dir=temp_dir, delete=True) as temp_file:
             self.get_file(name, temp_file.name)
             with open(temp_file.name, "r") as f:
                 return f.read()

datatailr/excel/__init__.py

@@ -2,8 +2,8 @@
 
 # noqa: F401
 try:
-    from dt.excel import Addin
-    from dt.excel import Queue
+    from datatailr.excel.addin import Addin
+    from datatailr.excel.addin import Queue
 except ImportError:
 
     class DummyAddin:

datatailr/excel/addin.py

@@ -0,0 +1,169 @@
+"""
+Copyright (c) 2025 - Datatailr Inc.
+All Rights Reserved.
+
+This file is part of Datatailr and subject to the terms and conditions
+defined in 'LICENSE.txt'. Unauthorized copying and/or distribution
+of this file, in parts or full, via any medium is strictly prohibited.
+"""
+
+import os
+import sys
+import importlib
+import subprocess
+import inspect
+import numpy as np
+from dt.excel_base import Addin as AddinBase, Queue  # type: ignore
+
+
+def __progress__(queue, stop):
+    from time import sleep
+
+    bar = ["█", "██", "███", "████", "█████", "██████", "███████"]
+
+    count = 0
+    while True:
+        if stop.is_set():
+            return
+        queue.push(bar[count % len(bar)])
+        count += 1
+        sleep(0.25)
+
+
+def get_package_root(mod):
+    # Given module, e.g., dt.excel located at /opt/datatailr/python/dt/excel.py
+    # return entry for sys.path so it could be imported as a module.
+    # For the module above: /opt/datatailr/python
+    mod_path = os.path.abspath(mod.__file__)
+    mod_parts = mod.__name__.split(".")
+    for _ in range(len(mod_parts)):
+        mod_path = os.path.dirname(mod_path)
+    return mod_path
+
+
+class Addin(AddinBase):
+    def __init__(self, *args, **kwargs):
+        super(Addin, self).__init__(*args, **kwargs)
+
+    def run(self, port):
+        # Excel addin executable will try to import an object literally called "addin"
+        # from a module passed to dt-excel.sh as an argument. So to find which module
+        # to pass to dt-excel.sh, we walk the callstack until a module with "addin"
+        # object of type Addin is found. If not -- inform user about this requirement.
+        found_module = None
+        for frame_info in inspect.stack():
+            mod = inspect.getmodule(frame_info.frame)
+            if not mod or not hasattr(mod, "__name__"):
+                continue
+
+            temp_path = get_package_root(mod)
+            sys.path.insert(0, temp_path)
+            try:
+                imported_mod = importlib.import_module(mod.__name__)
+            finally:
+                sys.path.pop(0)
+
+            addin_obj = getattr(imported_mod, "addin", None)
+            if addin_obj is self or id(addin_obj) == id(self):
+                found_module = mod
+                break
+
+        if not found_module:
+            raise ValueError(
+                "'addin' not found. Please, use 'addin' as variable name for your Addin instance."
+            )
+
+        if found_module.__name__ != "__main__":
+            # addin.run was called from the initial python script (where __name__ == "__main__")
+            module_name = found_module.__name__
+            if found_module.__file__ is None:
+                raise ValueError(f"Module {found_module.__name__} has no __file__")
+            dir_name = os.path.dirname(os.path.abspath(found_module.__file__))
+        else:
+            # initial python script did not call addin.run() itself (e.g. it imported function that called addin.run)
+            filename = inspect.getsourcefile(found_module)
+            if filename is None:
+                raise ValueError(f"Cannot determine filename for module {found_module}")
+            module_name = os.path.splitext(os.path.basename(filename))[0]
+            dir_name = os.path.dirname(os.path.abspath(filename))
+
+        subprocess.run(
+            [
+                "bash",
+                "-c",
+                f'PYTHONPATH="{dir_name}:$PYTHONPATH" /opt/datatailr/bin/dt-excel.sh -n -H "localhost" -l -p {port} -w 8000 {module_name}',
+            ]
+        )
+
+    def expose(
+        self, description, help, volatile=False, streaming=False, progressbar=False
+    ):
+        if streaming and progressbar:
+            raise ValueError(
+                "you cannot specify progressbar and streaming at the same time"
+            )
+
+        def decorator(func):
+            signature = inspect.signature(func)
+
+            def wrapper(*args, **kwargs):
+                id = args[0]
+
+                for arg in signature.parameters.values():
+                    if streaming and arg.name == "queue":
+                        continue
+
+                    if not (
+                        isinstance(kwargs[arg.name], arg.annotation)
+                        or isinstance(kwargs[arg.name], np.ndarray)
+                    ):
+                        raise ValueError(
+                            "excel/python/dt/excel.py: Got argument of wrong type, expected %s or numpy.ndarray, got %s"
+                            % (arg.annotation, type(kwargs[arg.name]))
+                        )
+
+                if not streaming:
+                    if not progressbar:
+                        result = func(**kwargs)
+                        if hasattr(result, "tolist"):
+                            result = result.tolist()
+                        return result
+
+                    from threading import Event, Thread
+
+                    error = None
+                    queue = Queue(self.name.lower() + "." + func.__name__, id)
+                    stop = Event()
+                    thread = Thread(target=__progress__, args=(queue, stop))
+                    thread.start()
+                    try:
+                        result = func(**kwargs)
+                    except Exception as exception:
+                        error = str(exception)
+
+                    stop.set()
+                    thread.join()
+
+                    if error is not None:
+                        queue.error(error)
+                    else:
+                        queue.push(result)
+                    return
+
+                try:
+                    func(Queue(self.name.lower() + "." + func.__name__, id), **kwargs)
+                except Exception as exception:
+                    queue.error(str(exception))
+
+            self.decorator_impl(
+                signature,
+                wrapper,
+                func.__name__,
+                description,
+                help,
+                volatile,
+                streaming or progressbar,
+            )
+            return wrapper
+
+        return decorator

datatailr/sbin/datatailr_run.py

@@ -35,6 +35,7 @@
 import concurrent.futures
 import subprocess
 import os
+import stat
 import shlex
 import sysconfig
 from typing import Optional, Tuple
@@ -80,6 +81,21 @@ def create_user_and_group() -> Tuple[str, str]:
     os.system(
         f"getent passwd {user} || useradd -g {group} -s /bin/bash -m {user} -u {uid} -o"
     )
+
+    permissions = (
+        stat.S_IWOTH
+        | stat.S_IXOTH
+        | stat.S_IWUSR
+        | stat.S_IRUSR
+        | stat.S_IRGRP
+        | stat.S_IWGRP
+        | stat.S_IXUSR
+        | stat.S_IXGRP
+    )
+
+    os.makedirs(f"/home/{user}/tmp/.dt", exist_ok=True)
+    os.chmod(f"/home/{user}/tmp/.dt", permissions)
+
     return user, group
 
 
@@ -88,16 +104,13 @@ def prepare_command_argv(command: str | list, user: str, env_vars: dict) -> list
         command = shlex.split(command)
 
     python_libdir = sysconfig.get_config_var("LIBDIR")
-    ld_library_path = get_env_var("LD_LIBRARY_PATH", "")
-
-    if ld_library_path:
-        python_libdir = ld_library_path + ":" + python_libdir
+    ld_library_path = get_env_var("LD_LIBRARY_PATH", None)
 
     # Base environment variables setup
     base_env = {
         "PATH": get_env_var("PATH", ""),
         "PYTHONPATH": get_env_var("PYTHONPATH", ""),
-        "LD_LIBRARY_PATH": python_libdir,
+        "LD_LIBRARY_PATH": ":".join(filter(None, [python_libdir, ld_library_path])),
     }
 
     merged_env = base_env | env_vars
@@ -144,7 +157,7 @@ def run_commands_in_parallel(
     user: str,
     env_vars: dict,
     log_stream_names: Optional[list[str | None]] = None,
-) -> tuple[int, int]:
+) -> int:
     """
     Executes two commands concurrently using a ThreadPoolExecutor.
     Returns a tuple of (return_code_cmd1, return_code_cmd2).
@@ -166,7 +179,7 @@ def run_commands_in_parallel(
         results = [
             future.result() for future in concurrent.futures.as_completed(futures)
         ]
-        return results[0], results[1]
+        return 0 if all(code == 0 for code in results) else 1
 
 
 def main():
@@ -226,6 +239,7 @@ def main():
             "--bind-addr=0.0.0.0:9090",
             f'--app-name="Datatailr IDE {get_env_var("DATATAILR_USER")}"',
         ]
+        job_name = get_env_var("DATATAILR_JOB_NAME")
         jupyter_command = [
             "jupyter-lab",
             "--ip='*'",
@@ -233,6 +247,9 @@ def main():
             "--no-browser",
             "--NotebookApp.token=''",
             "--NotebookApp.password=''",
+            f"--ServerApp.base_url=/workspace/{job_name}/jupyter/",
+            f"--ServerApp.static_url_prefix=/workspace/{job_name}/jupyter/static/",
+            f"--ServerApp.root_dir=/home/{user}",
         ]
         run_commands_in_parallel(
             [ide_command, jupyter_command], user, env, ["code-server", "jupyter"]

datatailr/scheduler/arguments_cache.py

@@ -49,10 +49,7 @@ class ArgumentsCache:
 
         :param use_persistent_cache: If True, use the persistent cache backend. Otherwise, use in-memory cache.
         """
-        try:
-            self.__bucket_name__ = dt__Tag().get("blob_storage_prefix") + "batch"
-        except Exception:
-            self.__bucket_name__ = "local-batch"
+        self.__bucket_name__ = dt__Tag().get("blob_storage_prefix") + "batch"
         self.use_persistent_cache = use_persistent_cache
         if not self.use_persistent_cache:
             # Create a temp folder, for local caching

datatailr/scheduler/base.py

@@ -183,6 +183,9 @@ class Job:
                 build_script_pre=build_script_pre,
                 build_script_post=build_script_post,
             )
+            if entrypoint is not None:
+                image.path_to_repo = entrypoint.path_to_repo
+                image.path_to_module = entrypoint.path_to_module
         self.image = image
         self.type = type if entrypoint is None else entrypoint.type
         self.entrypoint = entrypoint

datatailr/scheduler/batch.py

@@ -294,7 +294,7 @@ class BatchJob:
             env = {
                 "DATATAILR_BATCH_ID": str(self.dag.id),
                 "DATATAILR_JOB_ID": str(self.__id),
-                "DATATAILR_JOB_NAME": self.name,
+                "DATATAILR_JOB_NAME": f"{self.dag.name}[{self.__id}]",
             }
             self.entrypoint(env=env)
         else:
@@ -479,10 +479,26 @@ class Batch(Job):
 
     def prepare_args(self) -> None:
         def arg_name(arg: Union[BatchJob, str]) -> str:
-            return arg.name if isinstance(arg, BatchJob) else arg
+            return f"{self.name}[{arg.id}]" if isinstance(arg, BatchJob) else arg
+
+        def adjust_mapping(mapping: Dict[str, str]) -> Dict[str, str]:
+            result = {}
+            for k, v in mapping.items():
+                if isinstance(v, BatchJob):
+                    result[k] = f"{self.name}[{v.id}]"
+                elif isinstance(v, str):
+                    job = self.get_job_by_name(v)
+                    if job is not None:
+                        result[k] = f"{self.name}[{job.id}]"
+                    else:
+                        result[k] = v
+                else:
+                    raise TypeError(
+                        f"Unsupported type in argument mapping: {type(v)} for key {k}"
+                    )
+            return result
 
         def merged(dst: dict[str, str], src: dict[str, str]) -> dict[str, str]:
-            # copy so we don't mutate the original mapping
             out = dict(dst)
             seen_vals = set(out.values())
             for k, v in src.items():
@@ -492,12 +508,12 @@ class Batch(Job):
             return out
 
         args = {
-            j.name: merged(
-                j.argument_mapping, {k: arg_name(v) for k, v in j.args.items()}
+            f"{self.name}[{j.id}]": merged(
+                adjust_mapping(j.argument_mapping),
+                {j.argument_mapping.get(k, k): arg_name(v) for k, v in j.args.items()},
             )
             for j in self.__jobs
         }
-
         __ARGUMENTS_CACHE__.add_arguments(self.id, args)
 
     def save(self) -> Tuple[bool, str]:

{datatailr-0.1.70.dist-info → datatailr-0.1.72.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatailr
-Version: 0.1.70
+Version: 0.1.72
 Summary: Ready-to-Use Platform That Drives Business Insights
 Author-email: Datatailr <info@datatailr.com>
 License-Expression: MIT

{datatailr-0.1.70.dist-info → datatailr-0.1.72.dist-info}/RECORD

@@ -1,6 +1,6 @@
 datatailr/__init__.py,sha256=QTTG8X76BnlQwVx5N4ZQtSbLkgFipZ9NJGAbvtfuk_g,1051
 datatailr/acl.py,sha256=7hBwF7TP_ADoDryYEFuXx2FCLavLmp3k_F0-sEXg26g,4173
-datatailr/blob.py,sha256=9lKZKm4eRKVE4_t2zy3hh6Z_Ov4iaTA3-tJFjnQKezg,3313
+datatailr/blob.py,sha256=FHAB90wpt0DgDsejo46iCtZ1N2d6QMpo19uY_7NX1t8,3581
 datatailr/dt_json.py,sha256=3xmTqDBk68oPl2UW8UVOYPaBw4lAsVg6nDLwcen5nuo,2252
 datatailr/errors.py,sha256=p_e4ao3sFEfz1g4LvEDqw6bVzHJPJSINLjJ8H6_PqOo,751
 datatailr/group.py,sha256=AC0nCA44eEWZCJCq2klPqkFg_995mS3C_wu5uSFFLtU,4426
@@ -12,23 +12,24 @@ datatailr/version.py,sha256=N9K8ZxlwFFSz8XSgbgaTWZY4k2J0JKfj698nZ_O2pIU,536
 datatailr/wrapper.py,sha256=45RrMeYIFFWJAtOlQZRe1fT9daeq4vFlj6nIajbewEY,8080
 datatailr/build/__init__.py,sha256=_dA7b4L6wsaAFaSxUoYSJ1oaRqDHDMR20kqoCocSOss,487
 datatailr/build/image.py,sha256=YC8ML-l-sj6TcIBY-DCx_vaeI_7SmL9fPFhHnuxzRh0,5509
-datatailr/excel/__init__.py,sha256=IpXEPIFuu8IG3uQ8k7FDHqiHnChtyp-jgE8Plx2cWXQ,656
-datatailr/sbin/datatailr_run.py,sha256=dfNPo7sToUq2KnYoB-yPBlXq6ojhJ3BQEViBguQXq54,9412
+datatailr/excel/__init__.py,sha256=wox5ltPeOYZcZoRDW4R6tJsfOjf-0WZM2_pGgltGjdo,682
+datatailr/excel/addin.py,sha256=at0S1cNHShCOCXAml1W2sJmJ5DdNroTN6Bp6KWnYZ94,6104
+datatailr/sbin/datatailr_run.py,sha256=m8FZLYYXw7LFrvZhBXKWaBL784EKz27Qk7Se0Vo-KVY,9967
 datatailr/sbin/datatailr_run_app.py,sha256=itF76XC2F4RK9s6bkoEppEiYwSLHK_5Jai3yvC-kFhY,1501
 datatailr/sbin/datatailr_run_batch.py,sha256=UWnp96j_G66R_Cape7Bb-rbK6UBLF7Y5_mTlWyGJAVQ,1818
 datatailr/sbin/datatailr_run_excel.py,sha256=BLWmvxpKEE_8vJhs8E4VWq07FOBof5tlow-AkIEXtHw,1470
 datatailr/sbin/datatailr_run_service.py,sha256=DO9LGOpz3CVZOJJRHb4ac7AgY_mLbXHGadSyVCeIknc,1212
 datatailr/scheduler/__init__.py,sha256=qydHYVtEP6SUWd2CQ6FRdTdRWNz3SbYPJy4FK_wOvMk,1772
-datatailr/scheduler/arguments_cache.py,sha256=UEy55T6-rMg__t8S2hpXCmvyZCFH4FyAdMOb_vNbcIw,6328
-datatailr/scheduler/base.py,sha256=OWRblRCmKVe1stN43J35_g-1oKH4qteU4lrDRezyMV4,16829
-datatailr/scheduler/batch.py,sha256=CQCH1wHhW1qx09J7iQNQleErJ4n0nssAbd6u9YS6FMY,17735
+datatailr/scheduler/arguments_cache.py,sha256=00OE0DhobYteBOnirjulO1ltgGBRamAdCO168O3_Zes,6236
+datatailr/scheduler/base.py,sha256=WWi_VnDxev0GG6QolF3Wtj-p_JS5t2CN9VALYPl1OYo,16994
+datatailr/scheduler/batch.py,sha256=ZhEf3YkXf1_ieV5ivk4-me60ov9v5r9f9BdkJw84i_0,18475
 datatailr/scheduler/batch_decorator.py,sha256=LqL1bsupWLn-YEQUvFJYae7R3ogrL5-VodyiiScrkRw,5806
 datatailr/scheduler/constants.py,sha256=5WWTsfwZ_BA8gVDOTa2AQX9DJ0NzfaWgtY3vrODS2-8,606
 datatailr/scheduler/schedule.py,sha256=0XJJen2nL1xplRs0Xbjwgq3T-0bFCOrJzkSALdio998,3741
 datatailr/scheduler/utils.py,sha256=up6oR2iwe6G52LkvgfO394xchXgCYNjOMGRQW3e8PQk,1082
-datatailr-0.1.70.dist-info/licenses/LICENSE,sha256=ikKP4_O-UD_b8FuNdKmbzTb6odd0JX085ZW_FAPN3VI,1066
-datatailr-0.1.70.dist-info/METADATA,sha256=D6yUGFiVL9fUxb_OU0Wu9J-4s6xfTEJ-qAyfceKc-U0,5146
-datatailr-0.1.70.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-datatailr-0.1.70.dist-info/entry_points.txt,sha256=YqXfk2At-olW4PUSRkqvy_O3Mbv7uTKCCPuAAiz3Qbg,312
-datatailr-0.1.70.dist-info/top_level.txt,sha256=75gntW0X_SKpqxLL6hAPipvpk28GAhJBvoyqN_HohWU,10
-datatailr-0.1.70.dist-info/RECORD,,
+datatailr-0.1.72.dist-info/licenses/LICENSE,sha256=ikKP4_O-UD_b8FuNdKmbzTb6odd0JX085ZW_FAPN3VI,1066
+datatailr-0.1.72.dist-info/METADATA,sha256=w-sZvCPuMwHr_mFMYq9xTMwg1i1TD3sZ4VGhroQL7aU,5146
+datatailr-0.1.72.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+datatailr-0.1.72.dist-info/entry_points.txt,sha256=YqXfk2At-olW4PUSRkqvy_O3Mbv7uTKCCPuAAiz3Qbg,312
+datatailr-0.1.72.dist-info/top_level.txt,sha256=75gntW0X_SKpqxLL6hAPipvpk28GAhJBvoyqN_HohWU,10
+datatailr-0.1.72.dist-info/RECORD,,