datatailr 0.1.10__tar.gz → 0.1.11__tar.gz

{datatailr-0.1.10/src/datatailr.egg-info → datatailr-0.1.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatailr
-Version: 0.1.10
+Version: 0.1.11
 Summary: Ready-to-Use Platform That Drives Business Insights
 Author-email: Datatailr <info@datatailr.com>
 License-Expression: MIT
@@ -104,5 +104,47 @@ with Batch(name="MY test DAG", local_run=True) as dag:
 Running this code will create a graph of jobs and execute it.
 Each node on the graph represents a job, which in turn is a call to a function decorated with `@batch()`.
 
+Since this is a local run then the execution of each node will happen sequentially in the same process.
+
+To take advantage of the datatailr platform and execute the graph at scale, you can run it using the job scheduler as presented in the next section.
+
+### Execution at Scale
+To execute the graph at scale, you can use the Datatailr job scheduler. This allows you to run your jobs in parallel, taking advantage of the underlying infrastructure.
+
+You will first need to separate your function definitions from the DAG definition. This means you should define your functions as a separate module, which can be imported into the DAG definition.
+
+
+```python
+# my_module.py
+
+from datatailr.scheduler import batch, Batch
+
+@batch()
+def func_no_args() -> str:
+    return "no_args"
+
+
+@batch()
+def func_with_args(a: int, b: float) -> str:
+    return f"args: {a}, {b}"
+```
+
+To use these functions in a batch job, you just need to import them and run in a DAG context:
+
+```python
+from my_module import func_no_args, func_with_args
+from datatailr.scheduler import Schedule
+
+schedule = Schedule(at_hour=0)
+
+with Batch(name="MY test DAG", schedule=schedule) as dag:
+    for n in range(2):
+        res1 = func_no_args().alias(f"func_{n}")
+        res2 = func_with_args(1, res1).alias(f"func_with_args_{n}")
+```
+
+This will submit the entire DAG for execution, and the scheduler will take care of running the jobs in parallel and managing the resources.
+The DAG in the example above will be scheduled to run daily at 00:00.
+
 ___
 Visit [our website](https://www.datatailr.com/) for more!

{datatailr-0.1.10 → datatailr-0.1.11}/README.md

@@ -67,5 +67,47 @@ with Batch(name="MY test DAG", local_run=True) as dag:
 Running this code will create a graph of jobs and execute it.
 Each node on the graph represents a job, which in turn is a call to a function decorated with `@batch()`.
 
+Since this is a local run then the execution of each node will happen sequentially in the same process.
+
+To take advantage of the datatailr platform and execute the graph at scale, you can run it using the job scheduler as presented in the next section.
+
+### Execution at Scale
+To execute the graph at scale, you can use the Datatailr job scheduler. This allows you to run your jobs in parallel, taking advantage of the underlying infrastructure.
+
+You will first need to separate your function definitions from the DAG definition. This means you should define your functions as a separate module, which can be imported into the DAG definition.
+
+
+```python
+# my_module.py
+
+from datatailr.scheduler import batch, Batch
+
+@batch()
+def func_no_args() -> str:
+    return "no_args"
+
+
+@batch()
+def func_with_args(a: int, b: float) -> str:
+    return f"args: {a}, {b}"
+```
+
+To use these functions in a batch job, you just need to import them and run in a DAG context:
+
+```python
+from my_module import func_no_args, func_with_args
+from datatailr.scheduler import Schedule
+
+schedule = Schedule(at_hour=0)
+
+with Batch(name="MY test DAG", schedule=schedule) as dag:
+    for n in range(2):
+        res1 = func_no_args().alias(f"func_{n}")
+        res2 = func_with_args(1, res1).alias(f"func_with_args_{n}")
+```
+
+This will submit the entire DAG for execution, and the scheduler will take care of running the jobs in parallel and managing the resources.
+The DAG in the example above will be scheduled to run daily at 00:00.
+
 ___
 Visit [our website](https://www.datatailr.com/) for more!

{datatailr-0.1.10 → datatailr-0.1.11}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "datatailr"
-version = "0.1.10"
+version = "0.1.11"
 description = "Ready-to-Use Platform That Drives Business Insights"
 readme = "README.md"
 requires-python = ">=3.9"

{datatailr-0.1.10 → datatailr-0.1.11/src/datatailr.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatailr
-Version: 0.1.10
+Version: 0.1.11
 Summary: Ready-to-Use Platform That Drives Business Insights
 Author-email: Datatailr <info@datatailr.com>
 License-Expression: MIT
@@ -104,5 +104,47 @@ with Batch(name="MY test DAG", local_run=True) as dag:
 Running this code will create a graph of jobs and execute it.
 Each node on the graph represents a job, which in turn is a call to a function decorated with `@batch()`.
 
+Since this is a local run then the execution of each node will happen sequentially in the same process.
+
+To take advantage of the datatailr platform and execute the graph at scale, you can run it using the job scheduler as presented in the next section.
+
+### Execution at Scale
+To execute the graph at scale, you can use the Datatailr job scheduler. This allows you to run your jobs in parallel, taking advantage of the underlying infrastructure.
+
+You will first need to separate your function definitions from the DAG definition. This means you should define your functions as a separate module, which can be imported into the DAG definition.
+
+
+```python
+# my_module.py
+
+from datatailr.scheduler import batch, Batch
+
+@batch()
+def func_no_args() -> str:
+    return "no_args"
+
+
+@batch()
+def func_with_args(a: int, b: float) -> str:
+    return f"args: {a}, {b}"
+```
+
+To use these functions in a batch job, you just need to import them and run in a DAG context:
+
+```python
+from my_module import func_no_args, func_with_args
+from datatailr.scheduler import Schedule
+
+schedule = Schedule(at_hour=0)
+
+with Batch(name="MY test DAG", schedule=schedule) as dag:
+    for n in range(2):
+        res1 = func_no_args().alias(f"func_{n}")
+        res2 = func_with_args(1, res1).alias(f"func_with_args_{n}")
+```
+
+This will submit the entire DAG for execution, and the scheduler will take care of running the jobs in parallel and managing the resources.
+The DAG in the example above will be scheduled to run daily at 00:00.
+
 ___
 Visit [our website](https://www.datatailr.com/) for more!

{datatailr-0.1.10 → datatailr-0.1.11}/src/sbin/datatailr_run.py

@@ -111,10 +111,12 @@ def main():
         }
         run_command_as_user("datatailr_run_batch", user, env)
     elif job_type == "service":
+        port = get_env_var("DATATAILR_SERVICE_PORT")
         env = {
             "DATATAILR_JOB_NAME": job_name,
             "DATATAILR_JOB_ID": job_id,
             "DATATAILR_ENTRYPOINT": entrypoint,
+            "DATATAILR_SERVICE_PORT": port,
         }
         run_command_as_user("datatailr_run_service", user, env)
     elif job_type == "app":
@@ -125,10 +127,12 @@ def main():
         }
         run_command_as_user("datatailr_run_app", user, env)
     elif job_type == "excel":
+        host = get_env_var("DATATAILR_HOST")
         env = {
             "DATATAILR_JOB_NAME": job_name,
             "DATATAILR_JOB_ID": job_id,
             "DATATAILR_ENTRYPOINT": entrypoint,
+            "DATATAILR_HOST": host,
         }
         run_command_as_user("datatailr_run_excel", user, env)
     elif job_type == "IDE":

{datatailr-0.1.10 → datatailr-0.1.11}/src/sbin/datatailr_run_service.py

@@ -30,5 +30,5 @@ def run():
         raise ValueError("Environment variable 'DATATAILR_SERVICE_PORT' is not set.")
 
     entrypoint_module = importlib.import_module(entrypoint)
-    entrypoint_module.__service_main__(int(port))
     logger.info(f"Running entrypoint: {entrypoint}")
+    entrypoint_module.__service_main__(int(port))