datatailr 0.1.6__tar.gz → 0.1.8__tar.gz

{datatailr-0.1.6/src/datatailr.egg-info → datatailr-0.1.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatailr
-Version: 0.1.6
+Version: 0.1.8
 Summary: Ready-to-Use Platform That Drives Business Insights
 Author-email: Datatailr <info@datatailr.com>
 License-Expression: MIT
@@ -26,15 +26,27 @@ Requires-Dist: mypy; extra == "dev"
 Requires-Dist: types-setuptools; extra == "dev"
 Requires-Dist: toml; extra == "dev"
 Requires-Dist: coverage; extra == "dev"
+Requires-Dist: sphinx-rtd-theme; extra == "dev"
+Requires-Dist: sphinx; extra == "dev"
+Requires-Dist: sphinx-autodoc-typehints; extra == "dev"
+Requires-Dist: sphinx-autosummary; extra == "dev"
+Requires-Dist: sphinx-design; extra == "dev"
+Requires-Dist: sphinx-copybutton; extra == "dev"
+Requires-Dist: myst-parser; extra == "dev"
 Dynamic: license-file
 
-[![Datatailr Logo](https://framerusercontent.com/images/6CmFcG0o2ryoa5cMuDsvwESVWA.svg)](https://www.datatailr.com/)
+<div style="text-align: center;">
+  <a href="https://www.datatailr.com/" target="_blank">
+    <img src="https://s3.eu-west-1.amazonaws.com/datatailr.com/assets/datatailr-logo.svg" alt="Datatailr Logo" />
+  </a>
+</div>
+
+---
 
-___
 **Datatailr empowers your team to streamline analytics and data workflows
 from idea to production without infrastructure hurdles.**
 
-## What is Datatailr?
+# What is Datatailr?
 
 Datatailr is a platform that simplifies the process of building and deploying data applications.
 
@@ -69,7 +81,28 @@ print(datatailr.__provider__)
 
 
 ## Quickstart
+The following example shows how to create a simple data pipeline using the Datatailr Python package.
+
+```python
+from datatailr.scheduler import batch, Batch
+
+@batch()
+def func_no_args() -> str:
+    return "no_args"
+
+
+@batch()
+def func_with_args(a: int, b: float) -> str:
+    return f"args: {a}, {b}"
+
+with Batch(name="MY test DAG", local_run=True) as dag:
+    for n in range(2):
+        res1 = func_no_args().alias(f"func_{n}")
+        res2 = func_with_args(1, res1).alias(f"func_with_args_{n}")
+```
 
+Running this code will create a graph of jobs and execute it.
+Each node on the graph represents a job, which in turn is a call to a function decorated with `@batch()`.
 
 ___
 Visit [our website](https://www.datatailr.com/) for more!

{datatailr-0.1.6 → datatailr-0.1.8}/README.md

@@ -1,10 +1,15 @@
-[![Datatailr Logo](https://framerusercontent.com/images/6CmFcG0o2ryoa5cMuDsvwESVWA.svg)](https://www.datatailr.com/)
+<div style="text-align: center;">
+  <a href="https://www.datatailr.com/" target="_blank">
+    <img src="https://s3.eu-west-1.amazonaws.com/datatailr.com/assets/datatailr-logo.svg" alt="Datatailr Logo" />
+  </a>
+</div>
+
+---
 
-___
 **Datatailr empowers your team to streamline analytics and data workflows
 from idea to production without infrastructure hurdles.**
 
-## What is Datatailr?
+# What is Datatailr?
 
 Datatailr is a platform that simplifies the process of building and deploying data applications.
 
@@ -39,7 +44,28 @@ print(datatailr.__provider__)
 
 
 ## Quickstart
+The following example shows how to create a simple data pipeline using the Datatailr Python package.
+
+```python
+from datatailr.scheduler import batch, Batch
+
+@batch()
+def func_no_args() -> str:
+    return "no_args"
+
+
+@batch()
+def func_with_args(a: int, b: float) -> str:
+    return f"args: {a}, {b}"
+
+with Batch(name="MY test DAG", local_run=True) as dag:
+    for n in range(2):
+        res1 = func_no_args().alias(f"func_{n}")
+        res2 = func_with_args(1, res1).alias(f"func_with_args_{n}")
+```
 
+Running this code will create a graph of jobs and execute it.
+Each node on the graph represents a job, which in turn is a call to a function decorated with `@batch()`.
 
 ___
 Visit [our website](https://www.datatailr.com/) for more!

{datatailr-0.1.6 → datatailr-0.1.8}/pyproject.toml

@@ -1,10 +1,10 @@
 [build-system]
-requires = ["setuptools>=60", "wheel", "toml"]
+requires = ["setuptools", "wheel", "toml"]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "datatailr"
-version = "0.1.6"
+version = "0.1.8"
 description = "Ready-to-Use Platform That Drives Business Insights"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -34,7 +34,9 @@ homepage = "https://www.datatailr.com/"
 documentation = "https://docs.datatailr.com/"
 
 [project.scripts]
-run_dt_job = "datatailr.sbin.run_job:main"
+datatailr_run = "datatailr.sbin.datatailr_run:main"
+datatailr_run_batch = "datatailr.sbin.datatailr_run_batch:run"
+datatailr_run_app = "datatailr.sbin.datatailr_run_app:run"
 
 [project.optional-dependencies]
 dev = [
@@ -44,6 +46,13 @@ dev = [
     "types-setuptools",
     "toml",
     "coverage",
+    "sphinx-rtd-theme",
+    "sphinx",
+    "sphinx-autodoc-typehints",
+    "sphinx-autosummary",
+    "sphinx-design",
+    "sphinx-copybutton",
+    "myst-parser"
 ]
 
 [tool.ruff]
@@ -51,6 +60,7 @@ src = [
     "src",
     "../../tests/src/python"
 ]
+
 lint.ignore = ["F841"]
 show-fixes = true
 fix = true

datatailr-0.1.8/setup.py

@@ -0,0 +1,19 @@
+from datatailr import __version__  # type: ignore
+from setuptools import find_packages, setup
+
+setup(
+    name="datatailr",
+    version=__version__,
+    packages=find_packages(where="src", exclude=["test_module", "test_module.*"]),
+    package_dir={"": "src"},
+    data_files=[
+        (
+            "/datatailr/sbin",
+            [
+                "src/sbin/datatailr_run.py",
+                "src/sbin/datatailr_run_batch.py",
+                "src/sbin/datatailr_run_app.py",
+            ],
+        )
+    ],
+)

{datatailr-0.1.6 → datatailr-0.1.8}/src/datatailr/__init__.py

@@ -8,30 +8,12 @@
 #  of this file, in parts or full, via any medium is strictly prohibited.
 # *************************************************************************
 
-from datatailr.wrapper import (
-    dt__Blob,
-    dt__Dns,
-    dt__Email,
-    dt__Group,
-    dt__Job,
-    dt__Kv,
-    dt__Log,
-    dt__Node,
-    dt__Registry,
-    dt__Service,
-    dt__Settings,
-    dt__Sms,
-    dt__System,
-    dt__Tag,
-    dt__User,
-    mock_cli_tool,
-)
+from datatailr.wrapper import dt__System, mock_cli_tool
 from datatailr.group import Group
 from datatailr.user import User
 from datatailr.acl import ACL
 from datatailr.blob import Blob
 from datatailr.build import Image
-from datatailr.dt_json import dt_json
 from datatailr.utils import Environment, is_dt_installed
 from datatailr.version import __version__
 
@@ -50,21 +32,5 @@ __all__ = [
     "User",
     "__version__",
     "__provider__",
-    "dt__Blob",
-    "dt__Dns",
-    "dt__Email",
-    "dt__Group",
-    "dt__Job",
-    "dt__Kv",
-    "dt__Log",
-    "dt__Node",
-    "dt__Registry",
-    "dt__Service",
-    "dt__Settings",
-    "dt__Sms",
-    "dt__System",
-    "dt__Tag",
-    "dt__User",
-    "dt_json",
     "is_dt_installed",
 ]

{datatailr-0.1.6 → datatailr-0.1.8}/src/datatailr/acl.py

@@ -8,6 +8,8 @@
 #  of this file, in parts or full, via any medium is strictly prohibited.
 # *************************************************************************
 
+from __future__ import annotations
+
 import json
 from typing import List, Optional, Union
 
@@ -40,9 +42,9 @@ class ACL:
         self.__user_can_write = False
         self.__world_readable = False
 
-        self.__parse_permissions()
+        self.__parse_permissions_string()
 
-    def __parse_permissions(self):
+    def __parse_permissions_string(self):
         """
         Parse the permissions and set the corresponding flags.
         """
@@ -59,6 +61,19 @@ class ACL:
         self.__world_readable = self.permissions[4] == "r"
         self.__world_writable = self.permissions[5] == "w"
 
+    def _set_permissions_string(self):
+        """
+        Set the permissions string based on the current flags.
+        """
+        self.permissions = (
+            f"{'r' if self.__user_can_read else '-'}"
+            f"{'w' if self.__user_can_write else '-'}"
+            f"{'r' if self.__group_can_read else '-'}"
+            f"{'w' if self.__group_can_write else '-'}"
+            f"{'r' if self.__world_readable else '-'}"
+            f"{'w' if self.__world_writable else '-'}"
+        )
+
     def __repr__(self):
         return (
             f"ACL(user={self.user}, group={self.group}, permissions={self.permissions})"
@@ -67,7 +82,7 @@ class ACL:
     def to_dict(self):
         return {
             "user": self.user.name,
-            "group": self.group.name if self.group else self.user.primary_group.name,
+            "group": self.group.name if self.group else "dtusers",
             "group_can_read": self.__group_can_read,
             "group_can_write": self.__group_can_write,
             "user_can_read": self.__user_can_read,
@@ -76,5 +91,22 @@ class ACL:
             "world_writable": self.__world_writable,
         }
 
+    @classmethod
+    def from_dict(cls, acl_dict: dict) -> ACL:
+        """
+        Create an ACL instance from a dictionary.
+        """
+        user = User(acl_dict["user"])
+        group = Group.get(acl_dict["group"]) if "group" in acl_dict else None
+        acl = cls(user=user, group=group)
+        acl.__group_can_read = acl_dict.get("group_can_read", False)
+        acl.__group_can_write = acl_dict.get("group_can_write", False)
+        acl.__user_can_read = acl_dict.get("user_can_read", False)
+        acl.__user_can_write = acl_dict.get("user_can_write", False)
+        acl.__world_readable = acl_dict.get("world_readable", False)
+        acl.__world_writable = acl_dict.get("world_writable", False)
+        acl._set_permissions_string()
+        return acl
+
     def to_json(self):
         return json.dumps(self.to_dict())

{datatailr-0.1.6 → datatailr-0.1.8}/src/datatailr/blob.py

@@ -12,12 +12,13 @@ from __future__ import annotations
 
 import tempfile
 
-from datatailr import dt__Blob
+from datatailr.wrapper import dt__Blob
 
+# Datatailr Blob API Client
+__client__ = dt__Blob()
 
-class Blob:
-    __blob_storage__ = dt__Blob()
 
+class Blob:
     def ls(self, path: str) -> list[str]:
         """
         List files in the specified path.
@@ -25,7 +26,7 @@ class Blob:
         :param path: The path to list files from.
         :return: A list of file names in the specified path.
         """
-        return self.__blob_storage__.ls(path)
+        return __client__.ls(path)
 
     def get_file(self, name: str, path: str):
         """
@@ -35,7 +36,7 @@ class Blob:
             name (str): The name of the blob to retrieve.
             path (str): The path to store the blob as a file.
         """
-        return self.__blob_storage__.cp(f"blob:///{name}", path)
+        return __client__.cp(f"blob://{name}", path)
 
     def put_file(self, name: str, path: str):
         """
@@ -45,7 +46,7 @@ class Blob:
             name (str): The name of the blob to create.
             path (str): The path of the local file to copy.
         """
-        return self.__blob_storage__.cp(path, f"blob:///{name}")
+        return __client__.cp(path, f"blob://{name}")
 
     def exists(self, name: str) -> bool:
         """
@@ -57,7 +58,7 @@ class Blob:
         Returns:
             bool: True if the blob exists, False otherwise.
         """
-        return self.__blob_storage__.exists(name)
+        return __client__.exists(name)
 
     def delete(self, name: str):
         """
@@ -66,7 +67,7 @@ class Blob:
         Args:
             name (str): The name of the blob to delete.
         """
-        return self.__blob_storage__.rm(name)
+        return __client__.rm(name)
 
     def get_blob(self, name: str):
         """
@@ -81,9 +82,9 @@ class Blob:
         # Since direct reading and writting of blobs is not implemented yet, we are using a temporary file.
         # This is a workaround to allow reading the blob content directly from the blob storage.
 
-        with tempfile.NamedTemporaryFile(delete=False) as temp_file:
+        with tempfile.NamedTemporaryFile(delete=True) as temp_file:
             self.get_file(name, temp_file.name)
-            with open(temp_file.name, "rb") as f:
+            with open(temp_file.name, "r") as f:
                 return f.read()
 
     def put_blob(self, name: str, blob):
@@ -96,8 +97,7 @@ class Blob:
         """
         # Since direct reading and writting of blobs is not implemented yet, we are using a temporary file.
         # This is a workaround to allow writing the blob content directly to the blob storage.
-
-        with tempfile.NamedTemporaryFile(delete=False) as temp_file:
-            with open(temp_file.name, "wb") as f:
+        with tempfile.NamedTemporaryFile(delete=True) as temp_file:
+            with open(temp_file.name, "w") as f:
                 f.write(blob)
             self.put_file(name, temp_file.name)

{datatailr-0.1.6 → datatailr-0.1.8}/src/datatailr/build/image.py

@@ -25,9 +25,13 @@ class Image:
     def __init__(
         self,
         acl: Optional[ACL] = None,
-        python_requirements: str = "",
+        python_requirements: str | list[str] = "",
         build_script_pre: str = "",
         build_script_post: str = "",
+        branch_name: Optional[str] = None,
+        commit_hash: Optional[str] = None,
+        path_to_repo: Optional[str] = None,
+        path_to_module: Optional[str] = None,
     ):
         if acl is None:
             signed_user = User.signed_user()
@@ -39,9 +43,11 @@ class Image:
             raise TypeError("acl must be an instance of ACL.")
         self.acl = acl or ACL(signed_user)
 
-        if os.path.isfile(python_requirements):
+        if isinstance(python_requirements, str) and os.path.isfile(python_requirements):
             with open(python_requirements, "r") as f:
                 python_requirements = f.read()
+        elif isinstance(python_requirements, list):
+            python_requirements = "\n".join(python_requirements)
         if not isinstance(python_requirements, str):
             raise TypeError(
                 "python_requirements must be a string or a file path to a requirements file."
@@ -65,10 +71,36 @@ class Image:
                 "build_script_post must be a string or a file path to a script file."
             )
         self.build_script_post = build_script_post
+        self.branch_name = branch_name
+        self.commit_hash = commit_hash
+        self.path_to_repo = path_to_repo
+        self.path_to_module = path_to_module
 
     def __repr__(self):
         return f"Image(acl={self.acl},)"
 
+    def update(self, **kwargs):
+        for key, value in kwargs.items():
+            if key == "acl" and not isinstance(value, ACL):
+                raise TypeError("acl must be an instance of ACL.")
+            elif key == "python_requirements" and not isinstance(value, str):
+                raise TypeError("python_requirements must be a string.")
+            elif key == "build_script_pre" and not isinstance(value, str):
+                raise TypeError("build_script_pre must be a string.")
+            elif key == "build_script_post" and not isinstance(value, str):
+                raise TypeError("build_script_post must be a string.")
+            elif (
+                key in ["branch_name", "commit_hash", "path_to_repo", "path_to_module"]
+                and value is not None
+                and not isinstance(value, str)
+            ):
+                raise TypeError(f"{key} must be a string or None.")
+            if key not in self.__dict__:
+                raise AttributeError(
+                    f"'{self.__class__.__name__}' object has no attribute '{key}'"
+                )
+            setattr(self, key, value)
+
     def to_dict(self):
         """
         Convert the Image instance to a dictionary representation.
@@ -78,6 +110,10 @@ class Image:
             "python_requirements": self.python_requirements,
             "build_script_pre": self.build_script_pre,
             "build_script_post": self.build_script_post,
+            "branch_name": self.branch_name,
+            "commit_hash": self.commit_hash,
+            "path_to_repo": self.path_to_repo,
+            "path_to_module": self.path_to_module,
         }
 
     def to_json(self):

{datatailr-0.1.6 → datatailr-0.1.8}/src/datatailr/dt_json.py

@@ -10,6 +10,7 @@
 
 import importlib
 import json
+import base64
 
 
 class dt_json:
@@ -40,3 +41,34 @@ class dt_json:
     @classmethod
     def load(cls, json_file):
         return json.load(json_file)
+
+
+def encode_json(obj) -> str:
+    """
+    Encode an object to a JSON string with class information.
+    """
+    json_str = json.dumps(
+        obj, default=lambda o: o.__dict__ if hasattr(o, "__dict__") else o
+    )
+    return base64.b64encode(json_str.encode("utf-8")).decode("utf-8")
+
+
+def is_base64(s: str) -> bool:
+    try:
+        # Try decoding the string
+        base64.b64decode(s, validate=True)
+        return True
+    except (ValueError, TypeError):
+        # If decoding fails, it's not valid Base64
+        return False
+
+
+def decode_json(encoded_str: str):
+    """
+    Decode a JSON string back to an object.
+    """
+    if is_base64(encoded_str):
+        json_str = base64.b64decode(encoded_str.encode("utf-8")).decode("utf-8")
+    else:
+        json_str = encoded_str
+    return json.loads(json_str)

datatailr-0.1.8/src/datatailr/errors.py

@@ -0,0 +1,27 @@
+##########################################################################
+#
+#  Copyright (c) 2025 - Datatailr Inc.
+#  All Rights Reserved.
+#
+#  This file is part of Datatailr and subject to the terms and conditions
+#  defined in 'LICENSE.txt'. Unauthorized copying and/or distribution
+#  of this file, in parts or full, via any medium is strictly prohibited.
+##########################################################################
+
+
+class DatatailrError(Exception):
+    """Base class for all DataTailr exceptions."""
+
+    pass
+
+
+class BatchJobError(DatatailrError):
+    """Exception raised for errors related to batch jobs."""
+
+    pass
+
+
+class ScheduleError(DatatailrError):
+    """Exception raised for errors related to scheduling."""
+
+    pass

{datatailr-0.1.6 → datatailr-0.1.8}/src/datatailr/group.py

@@ -10,17 +10,25 @@
 
 from typing import Optional, Union
 
-from datatailr import dt__Group, mock_cli_tool
+from datatailr.wrapper import dt__Group, mock_cli_tool
+
+
+# Datatailr Group API Client
+__client__ = dt__Group()
 
 
 class Group:
-    """Representing a Datatailr Group
+    """
+    Representing a Datatailr Group.
+
     This class provides methods to interact with the Datatailr Group API.
     It allows you to create, update, delete, and manage groups within the Datatailr platform.
+
     Attributes:
         name (str): The name of the group.
         members (list): A list of members in the group.
         group_id (int): The unique identifier for the group.
+
     Static Methods:
         add(name: str) -> 'Group':
             Create a new group with the specified name.
@@ -32,6 +40,7 @@ class Group:
             Remove a group by its name.
         exists(name: str) -> bool:
             Check if a group exists by its name.
+
     Instance Methods:
         add_users(usernames: list) -> None:
             Add users to the group.
@@ -39,9 +48,6 @@ class Group:
             Remove users from the group.
     """
 
-    # Datatailr Group API Client
-    __client__ = dt__Group()
-
     def __init__(self, name: str):
         self.__name: str = name
         self.__members: list = []
@@ -69,9 +75,9 @@ class Group:
     def __refresh__(self):
         if not self.name:
             raise ValueError("Name is not set. Cannot refresh group.")
-        if isinstance(Group.__client__, mock_cli_tool):
+        if isinstance(__client__, mock_cli_tool):
             return
-        group = Group.__client__.get(self.name)
+        group = __client__.get(self.name)
         if group:
             self.__name = group["name"]
             self.__members = group["members"]
@@ -99,33 +105,33 @@ class Group:
 
     @staticmethod
     def add(name: str) -> Optional["Group"]:
-        Group.__client__.add(name)
+        __client__.add(name)
         return Group.get(name)
 
     @staticmethod
     def ls() -> list:
-        groups = Group.__client__.ls()
+        groups = __client__.ls()
         return [Group.get(group["name"]) for group in groups]
 
     @staticmethod
     def remove(name: str) -> None:
-        Group.__client__.rm(name)
+        __client__.rm(name)
         return None
 
     @staticmethod
     def exists(name: str) -> bool:
-        return Group.__client__.exists(name)
+        return __client__.exists(name)
 
     def add_users(self, usernames: list) -> None:
         if not self.name:
             raise ValueError("Name is not set. Cannot add users.")
-        Group.__client__.add_users(self.name, ",".join(usernames))
+        __client__.add_users(self.name, ",".join(usernames))
         self.__refresh__()
 
     def remove_users(self, usernames: list) -> None:
         if not self.name:
             raise ValueError("Name is not set. Cannot remove users.")
-        Group.__client__.rm_users(self.name, ",".join(usernames))
+        __client__.rm_users(self.name, ",".join(usernames))
         self.__refresh__()
 
     def is_member(self, user) -> bool: