datatailr 0.1.66__tar.gz → 0.1.81__tar.gz

{datatailr-0.1.66/src/datatailr.egg-info → datatailr-0.1.81}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatailr
-Version: 0.1.66
+Version: 0.1.81
 Summary: Ready-to-Use Platform That Drives Business Insights
 Author-email: Datatailr <info@datatailr.com>
 License-Expression: MIT
@@ -84,25 +84,27 @@ print(datatailr.__provider__)
 The following example shows how to create a simple data pipeline using the Datatailr Python package.
 
 ```python
-from datatailr.scheduler import batch_job, Batch
+from datatailr import workflow, task
 
-@batch_job()
+@task()
 def func_no_args() -> str:
     return "no_args"
 
 
-@batch_job()
+@task()
 def func_with_args(a: int, b: float) -> str:
     return f"args: {a}, {b}"
 
-with Batch(name="MY test DAG", local_run=True) as dag:
+@workflow(name="MY test DAG")
+def my_workflow():
     for n in range(2):
         res1 = func_no_args().alias(f"func_{n}")
         res2 = func_with_args(1, res1).alias(f"func_with_args_{n}")
+my_workflow(local_run=True)
 ```
 
 Running this code will create a graph of jobs and execute it.
-Each node on the graph represents a job, which in turn is a call to a function decorated with `@batch_job()`.
+Each node on the graph represents a job, which in turn is a call to a function decorated with `@task()`.
 
 Since this is a local run then the execution of each node will happen sequentially in the same process.
 
@@ -117,14 +119,14 @@ You will first need to separate your function definitions from the DAG definitio
 ```python
 # my_module.py
 
-from datatailr.scheduler import batch_job
+from datatailr import task
 
-@batch_job()
+@task()
 def func_no_args() -> str:
     return "no_args"
 
 
-@batch_job()
+@task()
 def func_with_args(a: int, b: float) -> str:
     return f"args: {a}, {b}"
 ```
@@ -133,18 +135,20 @@ To use these functions in a batch job, you just need to import them and run in a
 
 ```python
 from my_module import func_no_args, func_with_args
-from datatailr.scheduler import Batch, Schedule
+from datatailr import workflow
 
-schedule = Schedule(at_hours=0)
-
-with Batch(name="MY test DAG", schedule=schedule) as dag:
+@workflow(name="MY test DAG")
+def my_workflow():
     for n in range(2):
         res1 = func_no_args().alias(f"func_{n}")
         res2 = func_with_args(1, res1).alias(f"func_with_args_{n}")
+
+schedule = Schedule(at_hours=0)
+my_workflow(schedule=schedule)
 ```
 
-This will submit the entire DAG for execution, and the scheduler will take care of running the jobs in parallel and managing the resources.
-The DAG in the example above will be scheduled to run daily at 00:00.
+This will submit the entire workflow for execution, and the scheduler will take care of running the jobs in parallel and managing the resources.
+The workflow in the example above will be scheduled to run daily at 00:00.
 
 ___
 Visit [our website](https://www.datatailr.com/) for more!

{datatailr-0.1.66 → datatailr-0.1.81}/README.md

@@ -47,25 +47,27 @@ print(datatailr.__provider__)
 The following example shows how to create a simple data pipeline using the Datatailr Python package.
 
 ```python
-from datatailr.scheduler import batch_job, Batch
+from datatailr import workflow, task
 
-@batch_job()
+@task()
 def func_no_args() -> str:
     return "no_args"
 
 
-@batch_job()
+@task()
 def func_with_args(a: int, b: float) -> str:
     return f"args: {a}, {b}"
 
-with Batch(name="MY test DAG", local_run=True) as dag:
+@workflow(name="MY test DAG")
+def my_workflow():
     for n in range(2):
         res1 = func_no_args().alias(f"func_{n}")
         res2 = func_with_args(1, res1).alias(f"func_with_args_{n}")
+my_workflow(local_run=True)
 ```
 
 Running this code will create a graph of jobs and execute it.
-Each node on the graph represents a job, which in turn is a call to a function decorated with `@batch_job()`.
+Each node on the graph represents a job, which in turn is a call to a function decorated with `@task()`.
 
 Since this is a local run then the execution of each node will happen sequentially in the same process.
 
@@ -80,14 +82,14 @@ You will first need to separate your function definitions from the DAG definitio
 ```python
 # my_module.py
 
-from datatailr.scheduler import batch_job
+from datatailr import task
 
-@batch_job()
+@task()
 def func_no_args() -> str:
     return "no_args"
 
 
-@batch_job()
+@task()
 def func_with_args(a: int, b: float) -> str:
     return f"args: {a}, {b}"
 ```
@@ -96,18 +98,20 @@ To use these functions in a batch job, you just need to import them and run in a
 
 ```python
 from my_module import func_no_args, func_with_args
-from datatailr.scheduler import Batch, Schedule
+from datatailr import workflow
 
-schedule = Schedule(at_hours=0)
-
-with Batch(name="MY test DAG", schedule=schedule) as dag:
+@workflow(name="MY test DAG")
+def my_workflow():
     for n in range(2):
         res1 = func_no_args().alias(f"func_{n}")
         res2 = func_with_args(1, res1).alias(f"func_with_args_{n}")
+
+schedule = Schedule(at_hours=0)
+my_workflow(schedule=schedule)
 ```
 
-This will submit the entire DAG for execution, and the scheduler will take care of running the jobs in parallel and managing the resources.
-The DAG in the example above will be scheduled to run daily at 00:00.
+This will submit the entire workflow for execution, and the scheduler will take care of running the jobs in parallel and managing the resources.
+The workflow in the example above will be scheduled to run daily at 00:00.
 
 ___
 Visit [our website](https://www.datatailr.com/) for more!

{datatailr-0.1.66 → datatailr-0.1.81}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "datatailr"
-version = "0.1.66"
+version = "0.1.81"
 description = "Ready-to-Use Platform That Drives Business Insights"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -39,6 +39,7 @@ datatailr_run_batch = "datatailr.sbin.datatailr_run_batch:run"
 datatailr_run_app = "datatailr.sbin.datatailr_run_app:run"
 datatailr_run_excel = "datatailr.sbin.datatailr_run_excel:run"
 datatailr_run_service = "datatailr.sbin.datatailr_run_service:run"
+datatailr = "datatailr.sbin.datatailr_cli:main"
 
 [project.optional-dependencies]
 dev = [
@@ -57,6 +58,10 @@ dev = [
     "myst-parser"
 ]
 
+[tool.coverage.run]
+branch = true
+source = ["./src/datatailr"]
+
 [tool.ruff]
 src = [
     "src",

{datatailr-0.1.66 → datatailr-0.1.81}/setup.py

@@ -10,12 +10,14 @@ setup(
         (
             "/datatailr/sbin",
             [
+                "src/sbin/datatailr_cli.py",
                 "src/sbin/datatailr_run.py",
                 "src/sbin/datatailr_run_batch.py",
                 "src/sbin/datatailr_run_app.py",
                 "src/sbin/datatailr_run_excel.py",
                 "src/sbin/datatailr_run_service.py",
             ],
-        )
+        ),
+        ("datatailr_demo", ["src/datatailr_demo/README.md"]),
     ],
 )

{datatailr-0.1.66 → datatailr-0.1.81}/src/datatailr/__init__.py

@@ -16,6 +16,14 @@ from datatailr.blob import Blob
 from datatailr.build import Image
 from datatailr.utils import Environment, is_dt_installed
 from datatailr.version import __version__
+from datatailr.scheduler import (
+    App,
+    Service,
+    ExcelAddin,
+    workflow,
+    task,
+    set_allow_unsafe_scheduling,
+)
 
 system = dt__System()
 if isinstance(system, mock_cli_tool):
@@ -33,4 +41,10 @@ __all__ = [
     "__version__",
     "__provider__",
     "is_dt_installed",
+    "App",
+    "Service",
+    "ExcelAddin",
+    "workflow",
+    "task",
+    "set_allow_unsafe_scheduling",
 ]

{datatailr-0.1.66 → datatailr-0.1.81}/src/datatailr/acl.py

@@ -25,15 +25,15 @@ class ACL:
         self,
         user: Union[User, str],
         group: Optional[Union[Group, str]] = None,
-        permissions: Optional[List[str]] = None,
+        permissions: Optional[List[str] | str] = None,
     ):
         if user is None:
             user = User.signed_user()
         self.user = user if isinstance(user, User) else User.get(user)
-        if self.user is not None:
-            self.group = (
-                group if group and isinstance(group, Group) else self.user.primary_group
-            )
+        if group is None:
+            group = self.user.primary_group
+        group = group if isinstance(group, Group) else Group.get(str(group))
+        self.group = group
         self.permissions = permissions or "rwr---"
 
         self.__group_can_read = False

{datatailr-0.1.66 → datatailr-0.1.81}/src/datatailr/blob.py

@@ -10,12 +10,14 @@
 
 from __future__ import annotations
 
+import os
 import tempfile
 
 from datatailr.wrapper import dt__Blob
 
 # Datatailr Blob API Client
 __client__ = dt__Blob()
+__user__ = os.getenv("USER", "root")
 
 
 class Blob:
@@ -81,8 +83,13 @@ class Blob:
         """
         # Since direct reading and writting of blobs is not implemented yet, we are using a temporary file.
         # This is a workaround to allow reading the blob content directly from the blob storage.
-
-        with tempfile.NamedTemporaryFile(delete=True) as temp_file:
+        temp_dir = f"/home/{__user__}/tmp"
+        if not os.path.exists(temp_dir):
+            temp_dir = "/tmp"
+        else:
+            temp_dir += "/.dt"
+            os.makedirs(temp_dir, exist_ok=True)
+        with tempfile.NamedTemporaryFile(dir=temp_dir, delete=True) as temp_file:
             self.get_file(name, temp_file.name)
             with open(temp_file.name, "r") as f:
                 return f.read()

{datatailr-0.1.66 → datatailr-0.1.81}/src/datatailr/build/image.py

@@ -10,7 +10,7 @@
 
 import json
 import os
-import re
+import sys
 from typing import Optional
 
 from datatailr import ACL, User
@@ -26,7 +26,7 @@ class Image:
     def __init__(
         self,
         acl: Optional[ACL] = None,
-        python_version: str = "3.12",
+        python_version: str = "auto",
         python_requirements: str | list[str] = "",
         build_script_pre: str = "",
         build_script_post: str = "",
@@ -56,8 +56,10 @@ class Image:
     def python_version(self, value: str):
         if not isinstance(value, str):
             raise TypeError("python_version must be a string.")
-        if not re.match(r"^\d+\.\d+(\.\d+)?$", value):
-            raise ValueError("Invalid python_version format. Expected format: X.Y[.Z]")
+        if value.lower() == "auto":
+            value = f"{sys.version_info.major}.{sys.version_info.minor}"
+        if value not in ["3.10", "3.11", "3.12", "3.13", "3.14"]:
+            raise ValueError(f"Invalid python_version: {value}")
         self._python_version = value
 
     @property

{datatailr-0.1.66 → datatailr-0.1.81}/src/datatailr/excel/__init__.py

@@ -2,8 +2,8 @@
 
 # noqa: F401
 try:
-    from dt.excel import Addin
-    from dt.excel import Queue
+    from datatailr.excel.addin import Addin
+    from datatailr.excel.addin import Queue
 except ImportError:
 
     class DummyAddin:

datatailr-0.1.81/src/datatailr/excel/addin.py

@@ -0,0 +1,201 @@
+"""
+Copyright (c) 2025 - Datatailr Inc.
+All Rights Reserved.
+
+This file is part of Datatailr and subject to the terms and conditions
+defined in 'LICENSE.txt'. Unauthorized copying and/or distribution
+of this file, in parts or full, via any medium is strictly prohibited.
+"""
+
+import os
+import sys
+import importlib
+import subprocess
+import inspect
+from urllib.parse import urlparse
+
+import numpy as np
+
+try:
+    from dt.excel_base import Addin as AddinBase, Queue  # type: ignore
+except ImportError as e:
+    from datatailr.excel.stubs import AddinBase, Queue
+
+
+def __progress__(queue, stop):
+    from time import sleep
+
+    bar = ["█", "██", "███", "████", "█████", "██████", "███████"]
+
+    count = 0
+    while True:
+        if stop.is_set():
+            return
+        queue.push(bar[count % len(bar)])
+        count += 1
+        sleep(0.25)
+
+
+def get_package_root(mod):
+    # Given module, e.g., dt.excel located at /opt/datatailr/python/dt/excel.py
+    # return entry for sys.path so it could be imported as a module.
+    # For the module above: /opt/datatailr/python
+    mod_path = os.path.abspath(mod.__file__)
+    mod_parts = mod.__name__.split(".")
+    for _ in range(len(mod_parts)):
+        mod_path = os.path.dirname(mod_path)
+    return mod_path
+
+
+def matches_annotation(value, annotation):
+    if isinstance(value, np.ndarray):
+        return True
+    if annotation is bool:
+        return isinstance(value, bool) or (type(value) is int and value in (0, 1))
+    if annotation is float:
+        return isinstance(value, float) or (type(value) is int)
+    return isinstance(value, annotation)
+
+
+def extract_hostname(url: str) -> str | None:
+    url = url if url else ""
+    if "://" not in url:
+        url = "//" + url
+    return urlparse(url).hostname
+
+
+class Addin(AddinBase):
+    def __init__(self, *args, **kwargs):
+        super(Addin, self).__init__(*args, **kwargs)
+        f = inspect.currentframe().f_back
+        mod = inspect.getmodule(f)
+        if mod is not None:
+            setattr(mod, "__dt_addin__", self)
+
+    def run(self, port, ws_port, ide=True):
+        # Excel addin executable will try to import an object literally called "addin"
+        # from a module passed to dt-excel.sh as an argument. So to find which module
+        # to pass to dt-excel.sh, we walk the callstack until a module with "addin"
+        # object of type Addin is found. If not -- inform user about this requirement.
+        found_module = None
+        for frame_info in inspect.stack():
+            mod = inspect.getmodule(frame_info.frame)
+            if not mod or not hasattr(mod, "__name__"):
+                continue
+
+            temp_path = get_package_root(mod)
+            sys.path.insert(0, temp_path)
+            try:
+                imported_mod = importlib.import_module(mod.__name__)
+            finally:
+                sys.path.pop(0)
+
+            addin_obj = getattr(imported_mod, "__dt_addin__", None)
+            if addin_obj is self or id(addin_obj) == id(self):
+                found_module = mod
+                break
+
+        if not found_module:
+            raise ValueError(
+                "'__dt_addin__' not found."
+            )
+
+        if found_module.__name__ != "__main__":
+            # addin.run was called from the initial python script (where __name__ == "__main__")
+            module_name = found_module.__name__
+            if found_module.__file__ is None:
+                raise ValueError(f"Module {found_module.__name__} has no __file__")
+            dir_name = os.path.dirname(os.path.abspath(found_module.__file__))
+        else:
+            # initial python script did not call addin.run() itself (e.g. it imported function that called addin.run)
+            filename = inspect.getsourcefile(found_module)
+            if filename is None:
+                raise ValueError(f"Cannot determine filename for module {found_module}")
+            module_name = os.path.splitext(os.path.basename(filename))[0]
+            dir_name = os.path.dirname(os.path.abspath(filename))
+
+        ide_flag = "-i" if ide else ""
+        hostname = extract_hostname(os.environ.get("VSCODE_PROXY_URI"))
+
+        subprocess.run(
+            [
+                "bash",
+                "-c",
+                f'PYTHONPATH="{dir_name}:$PYTHONPATH" /opt/datatailr/bin/dt-excel.sh {ide_flag} -n -H {hostname} -p {port} -w {ws_port} {module_name}',
+            ]
+        )
+
+    def expose(
+        self, description, help, volatile=False, streaming=False, progressbar=False
+    ):
+        if streaming and progressbar:
+            raise ValueError(
+                "you cannot specify progressbar and streaming at the same time"
+            )
+
+        def decorator(func):
+            signature = inspect.signature(func)
+
+            def wrapper(*args, **kwargs):
+                # TODO: check whether it's possible to use a kwarg instead so that a decorated function can
+                # be called directly from python code without requiring positional argument for _id
+                _id = args[0]
+
+                bound = signature.bind_partial(**kwargs)
+                bound.apply_defaults()
+                for arg in signature.parameters.values():
+                    if streaming and arg.name == "queue":
+                        continue
+
+                    if not matches_annotation(
+                        bound.arguments[arg.name], arg.annotation
+                    ):
+                        raise ValueError(
+                            "excel/python/dt/excel.py: Got argument of wrong type, expected %s or numpy.ndarray, got %s"
+                            % (arg.annotation, type(bound.arguments[arg.name]))
+                        )
+                queue = Queue(self.name.lower() + "." + func.__name__, _id)
+                if not streaming:
+                    if not progressbar:
+                        result = func(**kwargs)
+                        if hasattr(result, "tolist"):
+                            result = result.tolist()
+                        return result
+
+                    from threading import Event, Thread
+
+                    error = None
+
+                    stop = Event()
+                    thread = Thread(target=__progress__, args=(queue, stop))
+                    thread.start()
+                    try:
+                        result = func(**kwargs)
+                    except Exception as exception:
+                        error = str(exception)
+
+                    stop.set()
+                    thread.join()
+
+                    if error is not None:
+                        queue.error(error)
+                    else:
+                        queue.push(result)
+                    return
+                try:
+                    func(queue, **kwargs)
+                except Exception as exception:
+                    queue.error(str(exception))
+
+            self.decorator_impl(
+                signature,
+                wrapper,
+                func.__name__,
+                description,
+                help,
+                volatile,
+                streaming or progressbar,
+            )
+            return wrapper
+
+        return decorator

datatailr-0.1.81/src/datatailr/excel/stubs.py

@@ -0,0 +1,37 @@
+"""
+Copyright (c) 2025 - Datatailr Inc.
+All Rights Reserved.
+
+This file is part of Datatailr and subject to the terms and conditions
+defined in 'LICENSE.txt'. Unauthorized copying and/or distribution
+of this file, in parts or full, via any medium is strictly prohibited.
+"""
+
+
+class AddinBase:
+    def __init__(self, name, *args, **kwargs):
+        self.name = name
+
+    def decorator_impl(
+        self,
+        signature,
+        wrapper,
+        func_name,
+        description,
+        help,
+        volatile,
+        streaming,
+    ):
+        pass
+
+
+class Queue:
+    def __init__(self, name, _id):
+        self.name = name
+        self.id = _id
+
+    def push(self, value):
+        print(f"Queue {self.name} ({self.id}): {value}")
+
+    def error(self, message):
+        print(f"Queue {self.name} ({self.id}) Error: {message}")