datatailr 0.1.8__py3-none-any.whl → 0.1.11__py3-none-any.whl

datatailr/group.py

@@ -10,7 +10,7 @@
 
 from typing import Optional, Union
 
-from datatailr.wrapper import dt__Group, mock_cli_tool
+from datatailr.wrapper import dt__Group
 
 
 # Datatailr Group API Client
@@ -75,8 +75,6 @@ class Group:
     def __refresh__(self):
         if not self.name:
             raise ValueError("Name is not set. Cannot refresh group.")
-        if isinstance(__client__, mock_cli_tool):
-            return
         group = __client__.get(self.name)
         if group:
             self.__name = group["name"]

datatailr/logging.py

@@ -14,7 +14,7 @@ from logging import StreamHandler
 from logging.handlers import RotatingFileHandler
 from typing import Optional
 from datatailr import User
-from datatailr.wrapper import dt__Tag, mock_cli_tool
+from datatailr.wrapper import dt__Tag
 
 
 def get_log_level() -> int:
@@ -34,15 +34,9 @@ def get_log_level() -> int:
 
 
 tag = dt__Tag()
-if isinstance(tag, mock_cli_tool):
-    node_name = "local"
-    node_ip = "0.0.0.0"
-    job_name = "local_job"
-
-else:
-    node_name = tag.get("node_name")
-    node_ip = tag.get("node_ip")
-    job_name = os.getenv("DATATAILR_JOB_NAME", "unknown_job")
+node_name = tag.get("node_name") or "local"
+node_ip = tag.get("node_ip")
+job_name = os.getenv("DATATAILR_JOB_NAME", "unknown_job")
 
 try:
     user = User.signed_user().name

datatailr/sbin/datatailr_run.py

@@ -111,10 +111,12 @@ def main():
         }
         run_command_as_user("datatailr_run_batch", user, env)
     elif job_type == "service":
+        port = get_env_var("DATATAILR_SERVICE_PORT")
         env = {
             "DATATAILR_JOB_NAME": job_name,
             "DATATAILR_JOB_ID": job_id,
             "DATATAILR_ENTRYPOINT": entrypoint,
+            "DATATAILR_SERVICE_PORT": port,
         }
         run_command_as_user("datatailr_run_service", user, env)
     elif job_type == "app":
@@ -125,10 +127,12 @@ def main():
         }
         run_command_as_user("datatailr_run_app", user, env)
     elif job_type == "excel":
+        host = get_env_var("DATATAILR_HOST")
         env = {
             "DATATAILR_JOB_NAME": job_name,
             "DATATAILR_JOB_ID": job_id,
             "DATATAILR_ENTRYPOINT": entrypoint,
+            "DATATAILR_HOST": host,
         }
         run_command_as_user("datatailr_run_excel", user, env)
     elif job_type == "IDE":

datatailr/sbin/datatailr_run_app.py

@@ -11,18 +11,27 @@
 # *************************************************************************
 
 import os
+import sys
+import runpy
+from importlib.resources import files
 
 from datatailr.logging import DatatailrLogger
 
+
 logger = DatatailrLogger(os.path.abspath(__file__)).get_logger()
 
 
 def run():
     logger.info("Starting Datatailr app...")
     entrypoint = os.environ.get("DATATAILR_ENTRYPOINT")
+    if entrypoint is None or ":" not in entrypoint:
+        raise ValueError(
+            "Environment variable 'DATATAILR_ENTRYPOINT' is not in the format 'module_name:file_name'."
+        )
 
-    if entrypoint is None:
-        raise ValueError("Environment variable 'DATATAILR_ENTRYPOINT' is not set.")
+    module_name, file_name = entrypoint.split(":")
 
-    os.system(entrypoint)
+    script = files(module_name).joinpath(file_name)
+    sys.argv = ["streamlit", "run", str(script), *sys.argv[1:]]
     logger.info(f"Running entrypoint: {entrypoint}")
+    runpy.run_module("streamlit", run_name="__main__")

datatailr/sbin/datatailr_run_excel.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+
+# *************************************************************************
+#
+#  Copyright (c) 2025 - Datatailr Inc.
+#  All Rights Reserved.
+#
+#  This file is part of Datatailr and subject to the terms and conditions
+#  defined in 'LICENSE.txt'. Unauthorized copying and/or distribution
+#  of this file, in parts or full, via any medium is strictly prohibited.
+# *************************************************************************
+
+import os
+import subprocess
+
+from datatailr.logging import DatatailrLogger
+
+logger = DatatailrLogger(os.path.abspath(__file__)).get_logger()
+
+
+def run():
+    logger.info("Starting Datatailr excel add-in...")
+    entrypoint = os.environ.get("DATATAILR_ENTRYPOINT")
+    hostname = os.environ.get("DATATAILR_HOST")
+
+    if entrypoint is None:
+        raise ValueError("Environment variable 'DATATAILR_ENTRYPOINT' is not set.")
+
+    if hostname is None:
+        raise ValueError("Environment variable 'DATATAILR_HOST' is not set.")
+
+    entrypoint = f'./dt-excel.sh -n -H "{hostname}" -p 8080 "{entrypoint}"'
+    logger.info(f"Running entrypoint: {entrypoint}")
+    subprocess.run(entrypoint, shell=True)

datatailr/sbin/datatailr_run_service.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+
+# *************************************************************************
+#
+#  Copyright (c) 2025 - Datatailr Inc.
+#  All Rights Reserved.
+#
+#  This file is part of Datatailr and subject to the terms and conditions
+#  defined in 'LICENSE.txt'. Unauthorized copying and/or distribution
+#  of this file, in parts or full, via any medium is strictly prohibited.
+# *************************************************************************
+
+import os
+import importlib
+
+from datatailr.logging import DatatailrLogger
+
+logger = DatatailrLogger(os.path.abspath(__file__)).get_logger()
+
+
+def run():
+    logger.info("Starting Datatailr service...")
+    entrypoint = os.environ.get("DATATAILR_ENTRYPOINT")
+    port = os.environ.get("DATATAILR_SERVICE_PORT")
+
+    if entrypoint is None:
+        raise ValueError("Environment variable 'DATATAILR_ENTRYPOINT' is not set.")
+
+    if port is None:
+        raise ValueError("Environment variable 'DATATAILR_SERVICE_PORT' is not set.")
+
+    entrypoint_module = importlib.import_module(entrypoint)
+    logger.info(f"Running entrypoint: {entrypoint}")
+    entrypoint_module.__service_main__(int(port))

datatailr/scheduler/base.py

@@ -15,7 +15,6 @@ import importlib
 import inspect
 import json
 import os
-import subprocess
 import tempfile
 import uuid
 from dataclasses import dataclass
@@ -139,7 +138,7 @@ class Job:
         name: str,
         environment: Optional[Environment] = Environment.DEV,
         image: Optional[Image] = None,
-        run_as: Optional[Union[str, User]] = User.signed_user(),
+        run_as: Optional[Union[str, User]] = None,
         resources: Resources = Resources(memory="100m", cpu=1),
         acl: Optional[ACL] = None,
         python_requirements: str = "",
@@ -293,16 +292,7 @@ class Job:
                 "Please commit your changes before running the job."
             )
 
-        remote_commit = (
-            subprocess.run(
-                ("remote_commit = $(git ls-remote origin HEAD)"),
-                shell=True,
-                capture_output=True,
-                text=True,
-            )
-            .stdout.strip()
-            .split("\t")[0]
-        )
+        remote_commit = run_shell_command("git ls-remote origin HEAD")[0].split("\t")[0]
 
         if local_commit != remote_commit:
             raise RepoValidationError(

datatailr/scheduler/batch.py

@@ -302,7 +302,7 @@ class Batch(Job):
         environment: Optional[Environment] = Environment.DEV,
         schedule: Optional[Schedule] = None,
         image: Optional[Image] = None,
-        run_as: Optional[Union[str, User]] = User.signed_user(),
+        run_as: Optional[Union[str, User]] = None,
         resources: Resources = Resources(memory="100m", cpu=1),
         acl: Optional[ACL] = None,
         local_run: bool = False,
@@ -440,9 +440,9 @@ class Batch(Job):
     def get_schedule_args(self) -> Dict[str, Any]:
         if isinstance(self.__schedule, Schedule):
             args = {
-                "at_minute": self.__schedule.at_minutes,
+                "at_minutes": self.__schedule.at_minutes,
                 "every_minute": self.__schedule.every_minute,
-                "at_hour": self.__schedule.at_hours,
+                "at_hours": self.__schedule.at_hours,
                 "every_hour": self.__schedule.every_hour,
                 "weekdays": self.__schedule.weekdays,
                 "day_of_month": self.__schedule.day_of_month,

datatailr/user.py

@@ -9,10 +9,9 @@
 # *************************************************************************
 
 from __future__ import annotations
-import sys
 from typing import Optional
 
-from datatailr.wrapper import dt__User, mock_cli_tool
+from datatailr.wrapper import dt__User
 
 # Datatailr User API Client
 __client__ = dt__User()
@@ -93,10 +92,6 @@ class User:
     def __refresh__(self):
         if not self.name:
             raise ValueError("Name is not set. Cannot refresh user.")
-        if isinstance(__client__, mock_cli_tool) or any(
-            "unit" in arg for arg in sys.argv
-        ):
-            return
         user = __client__.get(self.name)
         if user:
             self.__name = user["name"]
@@ -149,14 +144,6 @@ class User:
 
     @staticmethod
     def signed_user() -> User:
-        if isinstance(__client__, mock_cli_tool) or any(
-            "unit" in arg for arg in sys.argv
-        ):
-            user = User(name="test_user")
-            user.__expiry__ = "mock_expiry"
-            user.__signature__ = "mock_signature"
-            return user
-
         user_signature_and_expiry = __client__.signed_user()
         if user_signature_and_expiry:
             user = User(name=user_signature_and_expiry["name"])

{datatailr-0.1.8.dist-info → datatailr-0.1.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatailr
-Version: 0.1.8
+Version: 0.1.11
 Summary: Ready-to-Use Platform That Drives Business Insights
 Author-email: Datatailr <info@datatailr.com>
 License-Expression: MIT
@@ -104,5 +104,47 @@ with Batch(name="MY test DAG", local_run=True) as dag:
 Running this code will create a graph of jobs and execute it.
 Each node on the graph represents a job, which in turn is a call to a function decorated with `@batch()`.
 
+Since this is a local run then the execution of each node will happen sequentially in the same process.
+
+To take advantage of the datatailr platform and execute the graph at scale, you can run it using the job scheduler as presented in the next section.
+
+### Execution at Scale
+To execute the graph at scale, you can use the Datatailr job scheduler. This allows you to run your jobs in parallel, taking advantage of the underlying infrastructure.
+
+You will first need to separate your function definitions from the DAG definition. This means you should define your functions as a separate module, which can be imported into the DAG definition.
+
+
+```python
+# my_module.py
+
+from datatailr.scheduler import batch, Batch
+
+@batch()
+def func_no_args() -> str:
+    return "no_args"
+
+
+@batch()
+def func_with_args(a: int, b: float) -> str:
+    return f"args: {a}, {b}"
+```
+
+To use these functions in a batch job, you just need to import them and run in a DAG context:
+
+```python
+from my_module import func_no_args, func_with_args
+from datatailr.scheduler import Schedule
+
+schedule = Schedule(at_hour=0)
+
+with Batch(name="MY test DAG", schedule=schedule) as dag:
+    for n in range(2):
+        res1 = func_no_args().alias(f"func_{n}")
+        res2 = func_with_args(1, res1).alias(f"func_with_args_{n}")
+```
+
+This will submit the entire DAG for execution, and the scheduler will take care of running the jobs in parallel and managing the resources.
+The DAG in the example above will be scheduled to run daily at 00:00.
+
 ___
 Visit [our website](https://www.datatailr.com/) for more!

{datatailr-0.1.8.dist-info → datatailr-0.1.11.dist-info}/RECORD

@@ -3,28 +3,30 @@ datatailr/acl.py,sha256=tlDy6VlHinSy5W1FbVxcNQNi7FliWUXy3ssIbzaPp28,4157
 datatailr/blob.py,sha256=xkXT6RZcMww4YfLVjOyqvvPxWc-Ku6fTJ_PeCXyBys4,3159
 datatailr/dt_json.py,sha256=3xmTqDBk68oPl2UW8UVOYPaBw4lAsVg6nDLwcen5nuo,2252
 datatailr/errors.py,sha256=p_e4ao3sFEfz1g4LvEDqw6bVzHJPJSINLjJ8H6_PqOo,751
-datatailr/group.py,sha256=34unhas6jQH_KgQSz7N42AIT-4wAHtS9T8x-_W7AkA8,4444
-datatailr/logging.py,sha256=IkucGsHfkxaiy8Ul4Qy6U3fHbqCq3mChTDoaBNovYeA,3403
-datatailr/user.py,sha256=uIRMDdR_vey2r0d7aVEoq5WHb6tKLP8jBtEPc2n-xBY,7046
+datatailr/group.py,sha256=ExsrkqUooAfFLWKvnkp1ZxisSJD1yCp9TKqoCXDCwhs,4360
+datatailr/logging.py,sha256=4Rsx3wf2tAr1334E2goBjhC877RwbUNaFgBlh902vU4,3270
+datatailr/user.py,sha256=o0k80JNahhWK4RqKXPLTuPQAcyN9i2fNKunh0MrdPBI,6612
 datatailr/utils.py,sha256=mqnnERMyHNAuAgFY4Ry4O4yW0ZjCRtJbjfI5fXVqt2s,1524
 datatailr/version.py,sha256=N9K8ZxlwFFSz8XSgbgaTWZY4k2J0JKfj698nZ_O2pIU,536
 datatailr/wrapper.py,sha256=K9ZD76cWey_ikA6C5sKejwRaYBDln4QMg-RcoRGiuFc,7991
 datatailr/build/__init__.py,sha256=_dA7b4L6wsaAFaSxUoYSJ1oaRqDHDMR20kqoCocSOss,487
 datatailr/build/image.py,sha256=xeYKPR6usg0pqJNpbhsYI8t_BB2iWxhY0KBNTWqpb_Q,4939
-datatailr/sbin/datatailr_run.py,sha256=3WuvzdRrRFA0nWX9v-nShwoqjOQSnvzgZRvRvWdKJPI,5756
-datatailr/sbin/datatailr_run_app.py,sha256=67Y3RQPCZ2N9LIErEdXOsnYQ_4UzTmQygjyIfMhEkyk,891
+datatailr/sbin/datatailr_run.py,sha256=0kCs5XZMBRc3M6lnoVguVMP09uQ9d0-NNl55NUyoH80,5934
+datatailr/sbin/datatailr_run_app.py,sha256=AOkutzv4DeKfWZs-ZBciAMKnK4A05SfkVf1ZJnSSFwA,1231
 datatailr/sbin/datatailr_run_batch.py,sha256=UWnp96j_G66R_Cape7Bb-rbK6UBLF7Y5_mTlWyGJAVQ,1818
+datatailr/sbin/datatailr_run_excel.py,sha256=Gr_QZgqJrwgRVD9_o4v-2tbvU-QMvNHL7xUvFGhftFc,1163
+datatailr/sbin/datatailr_run_service.py,sha256=R8eNLN2SGnMtyfLy3vq9isUHr3dRzeBqESTquNK9Iho,1156
 datatailr/scheduler/__init__.py,sha256=qydHYVtEP6SUWd2CQ6FRdTdRWNz3SbYPJy4FK_wOvMk,1772
 datatailr/scheduler/arguments_cache.py,sha256=CydYR9o2pqfa4KsPTA1mJSBN-0YF47Q6AmODm4zAJQ4,6254
-datatailr/scheduler/base.py,sha256=OkoMxTjSzpFIFikWRctN9kuZM85oJDbNET8xKe8YUQY,12558
-datatailr/scheduler/batch.py,sha256=CK57jS6MvFrSj14q9ZTrD15akoBdAeTJ130MFb_aIN4,16449
+datatailr/scheduler/base.py,sha256=UcIyProXBeUHunw2JCCgQTIa0G-hrLyQr1gOX8nhAOQ,12323
+datatailr/scheduler/batch.py,sha256=77iUufBqRBau9Ku4IivLOKqh-lknclEak8jg2YhsX3c,16437
 datatailr/scheduler/batch_decorator.py,sha256=LqL1bsupWLn-YEQUvFJYae7R3ogrL5-VodyiiScrkRw,5806
 datatailr/scheduler/constants.py,sha256=5WWTsfwZ_BA8gVDOTa2AQX9DJ0NzfaWgtY3vrODS2-8,606
 datatailr/scheduler/schedule.py,sha256=vzXaBBKMVJeCGD0VxsRPeW80sYReJ83XxWzDHVgLibY,3734
 datatailr/scheduler/utils.py,sha256=up6oR2iwe6G52LkvgfO394xchXgCYNjOMGRQW3e8PQk,1082
-datatailr-0.1.8.dist-info/licenses/LICENSE,sha256=ikKP4_O-UD_b8FuNdKmbzTb6odd0JX085ZW_FAPN3VI,1066
-datatailr-0.1.8.dist-info/METADATA,sha256=rviYJtuu-teMbR_toxDPyC3ANi5sGzWeFFxH1PhijSM,3608
-datatailr-0.1.8.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-datatailr-0.1.8.dist-info/entry_points.txt,sha256=VVBtNTDhrPMkcqrU3XjTHrfEdPQXwbJW9kgH1C-rT7U,186
-datatailr-0.1.8.dist-info/top_level.txt,sha256=75gntW0X_SKpqxLL6hAPipvpk28GAhJBvoyqN_HohWU,10
-datatailr-0.1.8.dist-info/RECORD,,
+datatailr-0.1.11.dist-info/licenses/LICENSE,sha256=ikKP4_O-UD_b8FuNdKmbzTb6odd0JX085ZW_FAPN3VI,1066
+datatailr-0.1.11.dist-info/METADATA,sha256=U6YY4cGUx61N2WKJzh97CQqWUlHAvuzpb4czK2Z1jGE,5118
+datatailr-0.1.11.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+datatailr-0.1.11.dist-info/entry_points.txt,sha256=YqXfk2At-olW4PUSRkqvy_O3Mbv7uTKCCPuAAiz3Qbg,312
+datatailr-0.1.11.dist-info/top_level.txt,sha256=75gntW0X_SKpqxLL6hAPipvpk28GAhJBvoyqN_HohWU,10
+datatailr-0.1.11.dist-info/RECORD,,

{datatailr-0.1.8.dist-info → datatailr-0.1.11.dist-info}/entry_points.txt

@@ -2,3 +2,5 @@
 datatailr_run = datatailr.sbin.datatailr_run:main
 datatailr_run_app = datatailr.sbin.datatailr_run_app:run
 datatailr_run_batch = datatailr.sbin.datatailr_run_batch:run
+datatailr_run_excel = datatailr.sbin.datatailr_run_excel:run
+datatailr_run_service = datatailr.sbin.datatailr_run_service:run