datatailr 0.1.70__tar.gz → 0.1.72__tar.gz

{datatailr-0.1.70/src/datatailr.egg-info → datatailr-0.1.72}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatailr
-Version: 0.1.70
+Version: 0.1.72
 Summary: Ready-to-Use Platform That Drives Business Insights
 Author-email: Datatailr <info@datatailr.com>
 License-Expression: MIT

{datatailr-0.1.70 → datatailr-0.1.72}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "datatailr"
-version = "0.1.70"
+version = "0.1.72"
 description = "Ready-to-Use Platform That Drives Business Insights"
 readme = "README.md"
 requires-python = ">=3.9"

{datatailr-0.1.70 → datatailr-0.1.72}/src/datatailr/blob.py

@@ -10,12 +10,14 @@
 
 from __future__ import annotations
 
+import os
 import tempfile
 
 from datatailr.wrapper import dt__Blob
 
 # Datatailr Blob API Client
 __client__ = dt__Blob()
+__user__ = os.getenv("USER", "root")
 
 
 class Blob:
@@ -81,8 +83,13 @@ class Blob:
         """
         # Since direct reading and writting of blobs is not implemented yet, we are using a temporary file.
         # This is a workaround to allow reading the blob content directly from the blob storage.
-
-        with tempfile.NamedTemporaryFile(delete=True) as temp_file:
+        temp_dir = f"/home/{__user__}/tmp"
+        if not os.path.exists(temp_dir):
+            temp_dir = "/tmp"
+        else:
+            temp_dir += "/.dt"
+            os.makedirs(temp_dir, exist_ok=True)
+        with tempfile.NamedTemporaryFile(dir=temp_dir, delete=True) as temp_file:
             self.get_file(name, temp_file.name)
             with open(temp_file.name, "r") as f:
                 return f.read()

{datatailr-0.1.70 → datatailr-0.1.72}/src/datatailr/excel/__init__.py

@@ -2,8 +2,8 @@
 
 # noqa: F401
 try:
-    from dt.excel import Addin
-    from dt.excel import Queue
+    from datatailr.excel.addin import Addin
+    from datatailr.excel.addin import Queue
 except ImportError:
 
     class DummyAddin:

datatailr-0.1.72/src/datatailr/excel/addin.py

@@ -0,0 +1,169 @@
+"""
+Copyright (c) 2025 - Datatailr Inc.
+All Rights Reserved.
+
+This file is part of Datatailr and subject to the terms and conditions
+defined in 'LICENSE.txt'. Unauthorized copying and/or distribution
+of this file, in parts or full, via any medium is strictly prohibited.
+"""
+
+import os
+import sys
+import importlib
+import subprocess
+import inspect
+import numpy as np
+from dt.excel_base import Addin as AddinBase, Queue  # type: ignore
+
+
+def __progress__(queue, stop):
+    from time import sleep
+
+    bar = ["█", "██", "███", "████", "█████", "██████", "███████"]
+
+    count = 0
+    while True:
+        if stop.is_set():
+            return
+        queue.push(bar[count % len(bar)])
+        count += 1
+        sleep(0.25)
+
+
+def get_package_root(mod):
+    # Given module, e.g., dt.excel located at /opt/datatailr/python/dt/excel.py
+    # return entry for sys.path so it could be imported as a module.
+    # For the module above: /opt/datatailr/python
+    mod_path = os.path.abspath(mod.__file__)
+    mod_parts = mod.__name__.split(".")
+    for _ in range(len(mod_parts)):
+        mod_path = os.path.dirname(mod_path)
+    return mod_path
+
+
+class Addin(AddinBase):
+    def __init__(self, *args, **kwargs):
+        super(Addin, self).__init__(*args, **kwargs)
+
+    def run(self, port):
+        # Excel addin executable will try to import an object literally called "addin"
+        # from a module passed to dt-excel.sh as an argument. So to find which module
+        # to pass to dt-excel.sh, we walk the callstack until a module with "addin"
+        # object of type Addin is found. If not -- inform user about this requirement.
+        found_module = None
+        for frame_info in inspect.stack():
+            mod = inspect.getmodule(frame_info.frame)
+            if not mod or not hasattr(mod, "__name__"):
+                continue
+
+            temp_path = get_package_root(mod)
+            sys.path.insert(0, temp_path)
+            try:
+                imported_mod = importlib.import_module(mod.__name__)
+            finally:
+                sys.path.pop(0)
+
+            addin_obj = getattr(imported_mod, "addin", None)
+            if addin_obj is self or id(addin_obj) == id(self):
+                found_module = mod
+                break
+
+        if not found_module:
+            raise ValueError(
+                "'addin' not found. Please, use 'addin' as variable name for your Addin instance."
+            )
+
+        if found_module.__name__ != "__main__":
+            # addin.run was called from the initial python script (where __name__ == "__main__")
+            module_name = found_module.__name__
+            if found_module.__file__ is None:
+                raise ValueError(f"Module {found_module.__name__} has no __file__")
+            dir_name = os.path.dirname(os.path.abspath(found_module.__file__))
+        else:
+            # initial python script did not call addin.run() itself (e.g. it imported function that called addin.run)
+            filename = inspect.getsourcefile(found_module)
+            if filename is None:
+                raise ValueError(f"Cannot determine filename for module {found_module}")
+            module_name = os.path.splitext(os.path.basename(filename))[0]
+            dir_name = os.path.dirname(os.path.abspath(filename))
+
+        subprocess.run(
+            [
+                "bash",
+                "-c",
+                f'PYTHONPATH="{dir_name}:$PYTHONPATH" /opt/datatailr/bin/dt-excel.sh -n -H "localhost" -l -p {port} -w 8000 {module_name}',
+            ]
+        )
+
+    def expose(
+        self, description, help, volatile=False, streaming=False, progressbar=False
+    ):
+        if streaming and progressbar:
+            raise ValueError(
+                "you cannot specify progressbar and streaming at the same time"
+            )
+
+        def decorator(func):
+            signature = inspect.signature(func)
+
+            def wrapper(*args, **kwargs):
+                id = args[0]
+
+                for arg in signature.parameters.values():
+                    if streaming and arg.name == "queue":
+                        continue
+
+                    if not (
+                        isinstance(kwargs[arg.name], arg.annotation)
+                        or isinstance(kwargs[arg.name], np.ndarray)
+                    ):
+                        raise ValueError(
+                            "excel/python/dt/excel.py: Got argument of wrong type, expected %s or numpy.ndarray, got %s"
+                            % (arg.annotation, type(kwargs[arg.name]))
+                        )
+
+                if not streaming:
+                    if not progressbar:
+                        result = func(**kwargs)
+                        if hasattr(result, "tolist"):
+                            result = result.tolist()
+                        return result
+
+                    from threading import Event, Thread
+
+                    error = None
+                    queue = Queue(self.name.lower() + "." + func.__name__, id)
+                    stop = Event()
+                    thread = Thread(target=__progress__, args=(queue, stop))
+                    thread.start()
+                    try:
+                        result = func(**kwargs)
+                    except Exception as exception:
+                        error = str(exception)
+
+                    stop.set()
+                    thread.join()
+
+                    if error is not None:
+                        queue.error(error)
+                    else:
+                        queue.push(result)
+                    return
+
+                try:
+                    func(Queue(self.name.lower() + "." + func.__name__, id), **kwargs)
+                except Exception as exception:
+                    queue.error(str(exception))
+
+            self.decorator_impl(
+                signature,
+                wrapper,
+                func.__name__,
+                description,
+                help,
+                volatile,
+                streaming or progressbar,
+            )
+            return wrapper
+
+        return decorator

{datatailr-0.1.70 → datatailr-0.1.72}/src/datatailr/scheduler/arguments_cache.py

@@ -49,10 +49,7 @@ class ArgumentsCache:
 
         :param use_persistent_cache: If True, use the persistent cache backend. Otherwise, use in-memory cache.
         """
-        try:
-            self.__bucket_name__ = dt__Tag().get("blob_storage_prefix") + "batch"
-        except Exception:
-            self.__bucket_name__ = "local-batch"
+        self.__bucket_name__ = dt__Tag().get("blob_storage_prefix") + "batch"
         self.use_persistent_cache = use_persistent_cache
         if not self.use_persistent_cache:
             # Create a temp folder, for local caching

{datatailr-0.1.70 → datatailr-0.1.72}/src/datatailr/scheduler/base.py

@@ -183,6 +183,9 @@ class Job:
                 build_script_pre=build_script_pre,
                 build_script_post=build_script_post,
             )
+            if entrypoint is not None:
+                image.path_to_repo = entrypoint.path_to_repo
+                image.path_to_module = entrypoint.path_to_module
         self.image = image
         self.type = type if entrypoint is None else entrypoint.type
         self.entrypoint = entrypoint

{datatailr-0.1.70 → datatailr-0.1.72}/src/datatailr/scheduler/batch.py

@@ -294,7 +294,7 @@ class BatchJob:
             env = {
                 "DATATAILR_BATCH_ID": str(self.dag.id),
                 "DATATAILR_JOB_ID": str(self.__id),
-                "DATATAILR_JOB_NAME": self.name,
+                "DATATAILR_JOB_NAME": f"{self.dag.name}[{self.__id}]",
             }
             self.entrypoint(env=env)
         else:
@@ -479,10 +479,26 @@ class Batch(Job):
 
     def prepare_args(self) -> None:
         def arg_name(arg: Union[BatchJob, str]) -> str:
-            return arg.name if isinstance(arg, BatchJob) else arg
+            return f"{self.name}[{arg.id}]" if isinstance(arg, BatchJob) else arg
+
+        def adjust_mapping(mapping: Dict[str, str]) -> Dict[str, str]:
+            result = {}
+            for k, v in mapping.items():
+                if isinstance(v, BatchJob):
+                    result[k] = f"{self.name}[{v.id}]"
+                elif isinstance(v, str):
+                    job = self.get_job_by_name(v)
+                    if job is not None:
+                        result[k] = f"{self.name}[{job.id}]"
+                    else:
+                        result[k] = v
+                else:
+                    raise TypeError(
+                        f"Unsupported type in argument mapping: {type(v)} for key {k}"
+                    )
+            return result
 
         def merged(dst: dict[str, str], src: dict[str, str]) -> dict[str, str]:
-            # copy so we don't mutate the original mapping
             out = dict(dst)
             seen_vals = set(out.values())
             for k, v in src.items():
@@ -492,12 +508,12 @@ class Batch(Job):
             return out
 
         args = {
-            j.name: merged(
-                j.argument_mapping, {k: arg_name(v) for k, v in j.args.items()}
+            f"{self.name}[{j.id}]": merged(
+                adjust_mapping(j.argument_mapping),
+                {j.argument_mapping.get(k, k): arg_name(v) for k, v in j.args.items()},
             )
             for j in self.__jobs
         }
-
         __ARGUMENTS_CACHE__.add_arguments(self.id, args)
 
     def save(self) -> Tuple[bool, str]:

{datatailr-0.1.70 → datatailr-0.1.72/src/datatailr.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatailr
-Version: 0.1.70
+Version: 0.1.72
 Summary: Ready-to-Use Platform That Drives Business Insights
 Author-email: Datatailr <info@datatailr.com>
 License-Expression: MIT

{datatailr-0.1.70 → datatailr-0.1.72}/src/datatailr.egg-info/SOURCES.txt

@@ -23,6 +23,7 @@ src/datatailr.egg-info/top_level.txt
 src/datatailr/build/__init__.py
 src/datatailr/build/image.py
 src/datatailr/excel/__init__.py
+src/datatailr/excel/addin.py
 src/datatailr/scheduler/__init__.py
 src/datatailr/scheduler/arguments_cache.py
 src/datatailr/scheduler/base.py

{datatailr-0.1.70 → datatailr-0.1.72}/src/sbin/datatailr_run.py

@@ -35,6 +35,7 @@
 import concurrent.futures
 import subprocess
 import os
+import stat
 import shlex
 import sysconfig
 from typing import Optional, Tuple
@@ -80,6 +81,21 @@ def create_user_and_group() -> Tuple[str, str]:
     os.system(
         f"getent passwd {user} || useradd -g {group} -s /bin/bash -m {user} -u {uid} -o"
     )
+
+    permissions = (
+        stat.S_IWOTH
+        | stat.S_IXOTH
+        | stat.S_IWUSR
+        | stat.S_IRUSR
+        | stat.S_IRGRP
+        | stat.S_IWGRP
+        | stat.S_IXUSR
+        | stat.S_IXGRP
+    )
+
+    os.makedirs(f"/home/{user}/tmp/.dt", exist_ok=True)
+    os.chmod(f"/home/{user}/tmp/.dt", permissions)
+
     return user, group
 
 
@@ -88,16 +104,13 @@ def prepare_command_argv(command: str | list, user: str, env_vars: dict) -> list
         command = shlex.split(command)
 
     python_libdir = sysconfig.get_config_var("LIBDIR")
-    ld_library_path = get_env_var("LD_LIBRARY_PATH", "")
-
-    if ld_library_path:
-        python_libdir = ld_library_path + ":" + python_libdir
+    ld_library_path = get_env_var("LD_LIBRARY_PATH", None)
 
     # Base environment variables setup
     base_env = {
         "PATH": get_env_var("PATH", ""),
         "PYTHONPATH": get_env_var("PYTHONPATH", ""),
-        "LD_LIBRARY_PATH": python_libdir,
+        "LD_LIBRARY_PATH": ":".join(filter(None, [python_libdir, ld_library_path])),
     }
 
     merged_env = base_env | env_vars
@@ -144,7 +157,7 @@ def run_commands_in_parallel(
     user: str,
     env_vars: dict,
     log_stream_names: Optional[list[str | None]] = None,
-) -> tuple[int, int]:
+) -> int:
     """
     Executes two commands concurrently using a ThreadPoolExecutor.
     Returns a tuple of (return_code_cmd1, return_code_cmd2).
@@ -166,7 +179,7 @@ def run_commands_in_parallel(
         results = [
             future.result() for future in concurrent.futures.as_completed(futures)
         ]
-        return results[0], results[1]
+        return 0 if all(code == 0 for code in results) else 1
 
 
 def main():
@@ -226,6 +239,7 @@ def main():
             "--bind-addr=0.0.0.0:9090",
             f'--app-name="Datatailr IDE {get_env_var("DATATAILR_USER")}"',
         ]
+        job_name = get_env_var("DATATAILR_JOB_NAME")
         jupyter_command = [
             "jupyter-lab",
             "--ip='*'",
@@ -233,6 +247,9 @@ def main():
             "--no-browser",
             "--NotebookApp.token=''",
             "--NotebookApp.password=''",
+            f"--ServerApp.base_url=/workspace/{job_name}/jupyter/",
+            f"--ServerApp.static_url_prefix=/workspace/{job_name}/jupyter/static/",
+            f"--ServerApp.root_dir=/home/{user}",
         ]
         run_commands_in_parallel(
             [ide_command, jupyter_command], user, env, ["code-server", "jupyter"]