datatailr 0.1.5__tar.gz → 0.1.6__tar.gz

{datatailr-0.1.5/src/datatailr.egg-info → datatailr-0.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatailr
-Version: 0.1.5
+Version: 0.1.6
 Summary: Ready-to-Use Platform That Drives Business Insights
 Author-email: Datatailr <info@datatailr.com>
 License-Expression: MIT
@@ -31,7 +31,7 @@ Dynamic: license-file
 [![Datatailr Logo](https://framerusercontent.com/images/6CmFcG0o2ryoa5cMuDsvwESVWA.svg)](https://www.datatailr.com/)
 
 ___
-**Datatailr: empowers your team to streamline analytics and data workflows
+**Datatailr empowers your team to streamline analytics and data workflows
 from idea to production without infrastructure hurdles.**
 
 ## What is Datatailr?

{datatailr-0.1.5 → datatailr-0.1.6}/README.md

@@ -1,7 +1,7 @@
 [![Datatailr Logo](https://framerusercontent.com/images/6CmFcG0o2ryoa5cMuDsvwESVWA.svg)](https://www.datatailr.com/)
 
 ___
-**Datatailr: empowers your team to streamline analytics and data workflows
+**Datatailr empowers your team to streamline analytics and data workflows
 from idea to production without infrastructure hurdles.**
 
 ## What is Datatailr?

{datatailr-0.1.5 → datatailr-0.1.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "datatailr"
-version = "0.1.5"
+version = "0.1.6"
 description = "Ready-to-Use Platform That Drives Business Insights"
 readme = "README.md"
 requires-python = ">=3.9"

{datatailr-0.1.5 → datatailr-0.1.6}/src/datatailr/group.py

@@ -10,7 +10,7 @@
 
 from typing import Optional, Union
 
-from datatailr import dt__Group
+from datatailr import dt__Group, mock_cli_tool
 
 
 class Group:
@@ -69,6 +69,8 @@ class Group:
     def __refresh__(self):
         if not self.name:
             raise ValueError("Name is not set. Cannot refresh group.")
+        if isinstance(Group.__client__, mock_cli_tool):
+            return
         group = Group.__client__.get(self.name)
         if group:
             self.__name = group["name"]

{datatailr-0.1.5 → datatailr-0.1.6}/src/datatailr/scheduler/arguments_cache.py

@@ -22,12 +22,16 @@ This module is for internal use of the datatailr package.
 """
 
 from collections import defaultdict
+import pickle
 from typing import Any, Dict
 
-from datatailr import is_dt_installed
+from datatailr import is_dt_installed, Blob
 from datatailr.scheduler import BatchJob
 
 
+__BLOB_STORAGE__ = Blob()
+
+
 class ArgumentsCache:
     def __init__(self, use_persistent_cache: bool = is_dt_installed()):
         """
@@ -49,7 +53,8 @@ class ArgumentsCache:
         :param arguments: Dictionary of arguments to store.
         """
         if self.use_persistent_cache and isinstance(job, str):
-            self._add_to_persistent_cache(batch_run_id, job, arguments)
+            path = f"{batch_run_id}/{job}/args"
+            self._add_to_persistent_cache(path, arguments)
         else:
             self.in_memory_cache[batch_run_id][job]["args"] = arguments
 
@@ -62,14 +67,21 @@ class ArgumentsCache:
         :return: Dictionary of arguments.
         """
         if self.use_persistent_cache and isinstance(job, str):
-            return self._get_from_persistent_cache(batch_run_id, job)
+            path = f"{batch_run_id}/{job}/args"
+            arg_keys = self._get_from_persistent_cache(path)
+            if not isinstance(arg_keys, dict):
+                raise TypeError(
+                    f"Expected a dictionary for arguments, got {type(arg_keys)}"
+                )
+        else:
+            arg_keys = (
+                self.in_memory_cache.get(batch_run_id, {})
+                .get(job, {})
+                .get("args", {})
+                .items()
+            )
         arguments = {}
-        for key, value in (
-            self.in_memory_cache.get(batch_run_id, {})
-            .get(job, {})
-            .get("args", {})
-            .items()
-        ):
+        for key, value in arg_keys:
             if isinstance(value, BatchJob):
                 arguments[key] = value.name
             else:
@@ -85,7 +97,8 @@ class ArgumentsCache:
         :param result: Result of the batch job.
         """
         if self.use_persistent_cache and isinstance(job, str):
-            self._add_to_persistent_cache(batch_run_id, job, {"result": result})
+            path = f"{batch_run_id}/{job}/result"
+            self._add_to_persistent_cache(path, result)
         else:
             self.in_memory_cache[batch_run_id][job]["result"] = result
 
@@ -98,29 +111,31 @@ class ArgumentsCache:
         :return: Result of the batch job.
         """
         if self.use_persistent_cache and isinstance(job, str):
-            return self._get_from_persistent_cache(batch_run_id, job).get("result")
+            path = f"{batch_run_id}/{job}/result"
+            return self._get_from_persistent_cache(path)
         return self.in_memory_cache[batch_run_id][job].get("result")
 
-    def _add_to_persistent_cache(
-        self, batch_run_id: str, job_name: str, arguments: Dict[str, Any]
-    ):
+    def _add_to_persistent_cache(self, path: str, blob: Any):
         """
         Add arguments to the persistent cache.
+        This method serializes the blob using pickle and stores it in the Blob storage.
+        :param path: Path in the Blob storage where the blob will be stored.
+        :param blob: The blob to store, typically a dictionary of arguments.
+        :raises TypeError: If the blob cannot be pickled.
 
-        :param batch_run_id: Identifier for the batch run.
-        :param job_name: Name of the job.
-        :param arguments: Dictionary of arguments to store.
         """
-        pass
+        __BLOB_STORAGE__.put_blob(
+            path, pickle.dumps(blob, protocol=pickle.HIGHEST_PROTOCOL)
+        )
 
-    def _get_from_persistent_cache(
-        self, batch_run_id: str, job_name: str
-    ) -> Dict[str, Any]:
+    def _get_from_persistent_cache(self, path: str) -> Any:
         """
         Retrieve arguments from the persistent cache.
 
-        :param batch_run_id: Identifier for the batch run.
-        :param job_name: Name of the job.
-        :return: Dictionary of arguments.
+        :param path: Path in the Blob storage where the blob is stored.
         """
-        return {}
+        try:
+            data = __BLOB_STORAGE__.get_blob(path)
+            return pickle.loads(data)
+        except (TypeError, EOFError):
+            return {}

{datatailr-0.1.5 → datatailr-0.1.6}/src/datatailr/scheduler/batch.py

@@ -109,6 +109,20 @@ class BatchJob:
         ), "All dependencies must be integers representing job IDs."
         self.dag.add_job(self)
 
+    def __call__(self, *args, **kwds) -> BatchJob:
+        """
+        Allows the BatchJob instance to be called like a function, returning itself.
+        This is useful for chaining or functional-style programming.
+        """
+        return self
+
+    @property
+    def id(self) -> int:
+        """
+        Returns the unique identifier of the BatchJob instance.
+        """
+        return self.__id
+
     def alias(self, name: str):
         """
         Set an alias for the BatchJob instance.

{datatailr-0.1.5 → datatailr-0.1.6}/src/datatailr/scheduler/batch_decorator.py

@@ -40,7 +40,7 @@ def batch_decorator(memory=DEFAULT_TASK_MEMORY, cpu=DEFAULT_TASK_CPU):
     This decorator can be used to wrap functions that should be executed as part of batch jobs.
     """
 
-    def decorator(func):
+    def decorator(func) -> BatchJob:
         spec = inspect.getfullargspec(func)
         signature = inspect.signature(func)
         varargs = spec.varargs
@@ -107,6 +107,7 @@ def batch_decorator(memory=DEFAULT_TASK_MEMORY, cpu=DEFAULT_TASK_CPU):
         else:
             setattr(module, "__batch_main__", {func.__name__: batch_main})
 
-        return batch_main
+        # The return type is a BatchJob, but we use type: ignore to avoid type checking issues
+        return batch_main  # type: ignore
 
     return decorator

{datatailr-0.1.5 → datatailr-0.1.6}/src/datatailr/user.py

@@ -10,7 +10,7 @@
 
 from typing import Optional
 
-from datatailr import dt__User
+from datatailr import dt__User, mock_cli_tool
 
 
 class User:
@@ -86,6 +86,8 @@ class User:
     def __refresh__(self):
         if not self.name:
             raise ValueError("Name is not set. Cannot refresh user.")
+        if isinstance(User.__client__, mock_cli_tool):
+            return
         user = self.__client__.get(self.name)
         if user:
             self.__name = user["name"]
@@ -138,6 +140,11 @@ class User:
 
     @staticmethod
     def signed_user() -> "User":
+        if isinstance(User.__client__, mock_cli_tool):
+            user = User(name="mock_user")
+            user.__expiry__ = "mock_expiry"
+            user.__signature__ = "mock_signature"
+            return user
         user_signature_and_expiry = User.__client__.signed_user()
         if user_signature_and_expiry:
             user = User(name=user_signature_and_expiry["name"])

{datatailr-0.1.5 → datatailr-0.1.6}/src/datatailr/wrapper.py

@@ -11,17 +11,23 @@
 import json
 import os
 import subprocess
+from typing import Union
 
 from datatailr.utils import is_dt_installed
 
+API_JSON_PATH: Union[str, None] = None
+
 if is_dt_installed() or os.path.exists("/opt/datatailr/etc/api.json"):
     API_JSON_PATH = os.path.join("/opt", "datatailr", "etc", "api.json")
     CLI_TOOL = "dt"
 else:
     # For running local tests, use api.json from the repo and a mock CLI tool
-    API_JSON_PATH = os.path.join(
-        os.path.dirname(__file__), "..", "..", "..", "..", "dt", "api.json"
-    )
+    import sys
+
+    if any("unittest" in arg for arg in sys.argv):
+        API_JSON_PATH = os.path.join(
+            os.path.dirname(__file__), "..", "..", "..", "..", "dt", "api.json"
+        )
     CLI_TOOL = "echo"
 
 
@@ -155,7 +161,7 @@ def create_class(cmd_name, command):
 
 
 # Load API JSON and create classes at import time
-if os.path.exists(API_JSON_PATH):
+if API_JSON_PATH and os.path.exists(API_JSON_PATH):
     with open(API_JSON_PATH, "r") as f:
         api = json.load(f)
 

{datatailr-0.1.5 → datatailr-0.1.6/src/datatailr.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatailr
-Version: 0.1.5
+Version: 0.1.6
 Summary: Ready-to-Use Platform That Drives Business Insights
 Author-email: Datatailr <info@datatailr.com>
 License-Expression: MIT
@@ -31,7 +31,7 @@ Dynamic: license-file
 [![Datatailr Logo](https://framerusercontent.com/images/6CmFcG0o2ryoa5cMuDsvwESVWA.svg)](https://www.datatailr.com/)
 
 ___
-**Datatailr: empowers your team to streamline analytics and data workflows
+**Datatailr empowers your team to streamline analytics and data workflows
 from idea to production without infrastructure hurdles.**
 
 ## What is Datatailr?