datatailr 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

datatailr/__init__.py

@@ -0,0 +1,63 @@
+# *************************************************************************
+#
+#  Copyright (c) 2025 - Datatailr Inc.
+#  All Rights Reserved.
+#
+#  This file is part of Datatailr and subject to the terms and conditions
+#  defined in 'LICENSE.txt'. Unauthorized copying and/or distribution
+#  of this file, in parts or full, via any medium is strictly prohibited.
+# *************************************************************************
+
+from datatailr.wrapper import (
+    dt__Blob,
+    dt__Dns,
+    dt__Email,
+    dt__Group,
+    dt__Job,
+    dt__Kv,
+    dt__Log,
+    dt__Node,
+    dt__Registry,
+    dt__Service,
+    dt__Settings,
+    dt__Sms,
+    dt__System,
+    dt__Tag,
+    dt__User,
+)
+from datatailr.group import Group
+from datatailr.user import User
+from datatailr.acl import ACL
+from datatailr.blob import Blob
+from datatailr.build import Image
+from datatailr.dt_json import dt_json
+from datatailr.utils import Environment, is_dt_installed
+from datatailr.version import __version__
+
+
+__all__ = [
+    "ACL",
+    "Blob",
+    "Environment",
+    "Group",
+    "Image",
+    "User",
+    "__version__",
+    "dt__Blob",
+    "dt__Dns",
+    "dt__Email",
+    "dt__Group",
+    "dt__Job",
+    "dt__Kv",
+    "dt__Log",
+    "dt__Node",
+    "dt__Registry",
+    "dt__Service",
+    "dt__Settings",
+    "dt__Sms",
+    "dt__System",
+    "dt__Tag",
+    "dt__User",
+    "dt_json",
+    "is_dt_installed",
+]

datatailr/acl.py

@@ -0,0 +1,80 @@
+# *************************************************************************
+#
+#  Copyright (c) 2025 - Datatailr Inc.
+#  All Rights Reserved.
+#
+#  This file is part of Datatailr and subject to the terms and conditions
+#  defined in 'LICENSE.txt'. Unauthorized copying and/or distribution
+#  of this file, in parts or full, via any medium is strictly prohibited.
+# *************************************************************************
+
+import json
+from typing import List, Optional, Union
+
+from datatailr import Group, User
+
+
+class ACL:
+    """
+    A class to represent an Access Control List (ACL) for managing permissions.
+    """
+
+    def __init__(
+        self,
+        user: Union[User, str],
+        group: Optional[Union[Group, str]] = None,
+        permissions: Optional[List[str]] = None,
+    ):
+        if user is None:
+            user = User.signed_user()
+        self.user = user if isinstance(user, User) else User.get(user)
+        if self.user is not None:
+            self.group = (
+                group if group and isinstance(group, Group) else self.user.primary_group
+            )
+        self.permissions = permissions or "rwr---"
+
+        self.__group_can_read = False
+        self.__group_can_write = False
+        self.__user_can_read = False
+        self.__user_can_write = False
+        self.__world_readable = False
+
+        self.__parse_permissions()
+
+    def __parse_permissions(self):
+        """
+        Parse the permissions and set the corresponding flags.
+        """
+        if isinstance(self.permissions, str):
+            self.permissions = list(self.permissions)
+        if len(self.permissions) != 6:
+            raise ValueError(
+                "Permissions must be a list of 6 characters. representing 'r', 'w', and '-' for user, group, and world."
+            )
+        self.__user_can_read = self.permissions[0] == "r"
+        self.__user_can_write = self.permissions[1] == "w"
+        self.__group_can_read = self.permissions[2] == "r"
+        self.__group_can_write = self.permissions[3] == "w"
+        self.__world_readable = self.permissions[4] == "r"
+        self.__world_writable = self.permissions[5] == "w"
+
+    def __repr__(self):
+        return (
+            f"ACL(user={self.user}, group={self.group}, permissions={self.permissions})"
+        )
+
+    def to_dict(self):
+        return {
+            "user": self.user.name,
+            "group": self.group.name if self.group else self.user.primary_group.name,
+            "group_can_read": self.__group_can_read,
+            "group_can_write": self.__group_can_write,
+            "user_can_read": self.__user_can_read,
+            "user_can_write": self.__user_can_write,
+            "world_readable": self.__world_readable,
+            "world_writable": self.__world_writable,
+        }
+
+    def to_json(self):
+        return json.dumps(self.to_dict())

datatailr/blob.py

@@ -0,0 +1,103 @@
+##########################################################################
+#
+#  Copyright (c) 2025 - Datatailr Inc.
+#  All Rights Reserved.
+#
+#  This file is part of Datatailr and subject to the terms and conditions
+#  defined in 'LICENSE.txt'. Unauthorized copying and/or distribution
+#  of this file, in parts or full, via any medium is strictly prohibited.
+##########################################################################
+
+from __future__ import annotations
+
+import tempfile
+
+from datatailr import dt__Blob
+
+
+class Blob:
+    __blob_storage__ = dt__Blob()
+
+    def ls(self, path: str) -> list[str]:
+        """
+        List files in the specified path.
+
+        :param path: The path to list files from.
+        :return: A list of file names in the specified path.
+        """
+        return self.__blob_storage__.ls(path)
+
+    def get_file(self, name: str, path: str):
+        """
+        Copy a blob file to a local file.
+
+        Args:
+            name (str): The name of the blob to retrieve.
+            path (str): The path to store the blob as a file.
+        """
+        return self.__blob_storage__.cp(f"blob:///{name}", path)
+
+    def put_file(self, name: str, path: str):
+        """
+        Copy a local file to a blob.
+
+        Args:
+            name (str): The name of the blob to create.
+            path (str): The path of the local file to copy.
+        """
+        return self.__blob_storage__.cp(path, f"blob:///{name}")
+
+    def exists(self, name: str) -> bool:
+        """
+        Check if a blob exists.
+
+        Args:
+            name (str): The name of the blob to check.
+
+        Returns:
+            bool: True if the blob exists, False otherwise.
+        """
+        return self.__blob_storage__.exists(name)
+
+    def delete(self, name: str):
+        """
+        Delete a blob.
+
+        Args:
+            name (str): The name of the blob to delete.
+        """
+        return self.__blob_storage__.rm(name)
+
+    def get_blob(self, name: str):
+        """
+        Get a blob object.
+
+        Args:
+            name (str): The name of the blob to retrieve.
+
+        Returns:
+            Blob: The blob object.
+        """
+        # Since direct reading and writting of blobs is not implemented yet, we are using a temporary file.
+        # This is a workaround to allow reading the blob content directly from the blob storage.
+
+        with tempfile.NamedTemporaryFile(delete=False) as temp_file:
+            self.get_file(name, temp_file.name)
+            with open(temp_file.name, "rb") as f:
+                return f.read()
+
+    def put_blob(self, name: str, blob):
+        """
+        Put a blob object into the blob storage.
+
+        Args:
+            name (str): The name of the blob to create.
+            blob: The blob object to store.
+        """
+        # Since direct reading and writting of blobs is not implemented yet, we are using a temporary file.
+        # This is a workaround to allow writing the blob content directly to the blob storage.
+
+        with tempfile.NamedTemporaryFile(delete=False) as temp_file:
+            with open(temp_file.name, "wb") as f:
+                f.write(blob)
+            self.put_file(name, temp_file.name)

datatailr/build/__init__.py

@@ -0,0 +1,11 @@
+# *************************************************************************
+#
+#  Copyright (c) 2025 - Datatailr Inc.
+#  All Rights Reserved.
+#
+#  This file is part of Datatailr and subject to the terms and conditions
+#  defined in 'LICENSE.txt'. Unauthorized copying and/or distribution
+#  of this file, in parts or full, via any medium is strictly prohibited.
+# *************************************************************************
+
+from datatailr.build.image import Image as Image

datatailr/build/image.py

@@ -0,0 +1,87 @@
+# *************************************************************************
+#
+#  Copyright (c) 2025 - Datatailr Inc.
+#  All Rights Reserved.
+#
+#  This file is part of Datatailr and subject to the terms and conditions
+#  defined in 'LICENSE.txt'. Unauthorized copying and/or distribution
+#  of this file, in parts or full, via any medium is strictly prohibited.
+# *************************************************************************
+
+import json
+import os
+from typing import Optional
+
+from datatailr import ACL, User
+
+
+class Image:
+    """
+    Represents a container image for a job.
+    The image is defined based on its' python dependencies and two 'build scripts' expressed as Dockerfile commands.
+    All attributes can be initialized with either a string or a file name.
+    """
+
+    def __init__(
+        self,
+        acl: Optional[ACL] = None,
+        python_requirements: str = "",
+        build_script_pre: str = "",
+        build_script_post: str = "",
+    ):
+        if acl is None:
+            signed_user = User.signed_user()
+            if signed_user is None:
+                raise ValueError(
+                    "ACL cannot be None. Please provide a valid ACL or ensure a user is signed in."
+                )
+        elif not isinstance(acl, ACL):
+            raise TypeError("acl must be an instance of ACL.")
+        self.acl = acl or ACL(signed_user)
+
+        if os.path.isfile(python_requirements):
+            with open(python_requirements, "r") as f:
+                python_requirements = f.read()
+        if not isinstance(python_requirements, str):
+            raise TypeError(
+                "python_requirements must be a string or a file path to a requirements file."
+            )
+        self.python_requirements = python_requirements
+
+        if os.path.isfile(build_script_pre):
+            with open(build_script_pre, "r") as f:
+                build_script_pre = f.read()
+        if not isinstance(build_script_pre, str):
+            raise TypeError(
+                "build_script_pre must be a string or a file path to a script file."
+            )
+        self.build_script_pre = build_script_pre
+
+        if os.path.isfile(build_script_post):
+            with open(build_script_post, "r") as f:
+                build_script_post = f.read()
+        if not isinstance(build_script_post, str):
+            raise TypeError(
+                "build_script_post must be a string or a file path to a script file."
+            )
+        self.build_script_post = build_script_post
+
+    def __repr__(self):
+        return f"Image(acl={self.acl},)"
+
+    def to_dict(self):
+        """
+        Convert the Image instance to a dictionary representation.
+        """
+        return {
+            "acl": self.acl.to_dict(),
+            "python_requirements": self.python_requirements,
+            "build_script_pre": self.build_script_pre,
+            "build_script_post": self.build_script_post,
+        }
+
+    def to_json(self):
+        """
+        Convert the Image instance to a JSON string representation.
+        """
+        return json.dumps(self.to_dict(), indent=4)

datatailr/dt_json.py

@@ -0,0 +1,42 @@
+# *************************************************************************
+#
+#  Copyright (c) 2025 - Datatailr Inc.
+#  All Rights Reserved.
+#
+#  This file is part of Datatailr and subject to the terms and conditions
+#  defined in 'LICENSE.txt'. Unauthorized copying and/or distribution
+#  of this file, in parts or full, via any medium is strictly prohibited.
+# *************************************************************************
+
+import importlib
+import json
+
+
+class dt_json:
+    @staticmethod
+    def dumps(obj):
+        d = obj.__dict__ if hasattr(obj, "__dict__") else obj
+        d["__class__"] = obj.__class__.__name__
+        return json.dumps(d)
+
+    @classmethod
+    def loads(cls, json_str):
+        d = json.loads(json_str)
+        class_name = d.get("__class__")
+        module = importlib.import_module("datatailr")
+        if class_name and hasattr(module, class_name):
+            target_class = getattr(module, class_name)
+            obj = target_class.__new__(target_class)
+            if hasattr(obj, "from_json"):
+                obj.from_json(d)
+                return obj
+            # fallback: set attributes directly
+            for k, v in d.items():
+                if k != "__class__":
+                    setattr(obj, k, v)
+            return obj
+        return d  # fallback to dict if not registered
+
+    @classmethod
+    def load(cls, json_file):
+        return json.load(json_file)

datatailr/errors.py

@@ -0,0 +1,10 @@
+class DatatailrError(Exception):
+    """Base class for all DataTailr exceptions."""
+
+    pass
+
+
+class BatchJobError(DatatailrError):
+    """Exception raised for errors related to batch jobs."""
+
+    pass

datatailr/group.py

@@ -0,0 +1,136 @@
+# *************************************************************************
+#
+#  Copyright (c) 2025 - Datatailr Inc.
+#  All Rights Reserved.
+#
+#  This file is part of Datatailr and subject to the terms and conditions
+#  defined in 'LICENSE.txt'. Unauthorized copying and/or distribution
+#  of this file, in parts or full, via any medium is strictly prohibited.
+# *************************************************************************
+
+from typing import Optional, Union
+
+from datatailr import dt__Group
+
+
+class Group:
+    """Representing a Datatailr Group
+    This class provides methods to interact with the Datatailr Group API.
+    It allows you to create, update, delete, and manage groups within the Datatailr platform.
+    Attributes:
+        name (str): The name of the group.
+        members (list): A list of members in the group.
+        group_id (int): The unique identifier for the group.
+    Static Methods:
+        add(name: str) -> 'Group':
+            Create a new group with the specified name.
+        get(name: str) -> 'Group':
+            Retrieve a group by its name.
+        list() -> list:
+            List all groups available in the Datatailr platform.
+        remove(name: str) -> None:
+            Remove a group by its name.
+        exists(name: str) -> bool:
+            Check if a group exists by its name.
+    Instance Methods:
+        add_users(usernames: list) -> None:
+            Add users to the group.
+        remove_users(usernames: list) -> None:
+            Remove users from the group.
+    """
+
+    # Datatailr Group API Client
+    __client__ = dt__Group()
+
+    def __init__(self, name: str):
+        self.__name: str = name
+        self.__members: list = []
+        self.__group_id: int | None = None
+
+        self.__refresh__()
+
+    def __repr__(self):
+        return (
+            f"Group(name={self.name}, members={self.members}, group_id={self.group_id})"
+        )
+
+    def __str__(self):
+        return f"<Group: {self.name} | {self.group_id}>"
+
+    def __eq__(self, other):
+        if not isinstance(other, Group):
+            return NotImplemented
+        return (
+            self.group_id == other.group_id
+            and self.name == other.name
+            and set(self.members) == set(other.members)
+        )
+
+    def __refresh__(self):
+        if not self.name:
+            raise ValueError("Name is not set. Cannot refresh group.")
+        group = Group.__client__.get(self.name)
+        if group:
+            self.__name = group["name"]
+            self.__members = group["members"]
+            self.__group_id = group["group_id"]
+        else:
+            raise ValueError(f"Group '{self.name}' does not exist.")
+
+    @property
+    def name(self) -> str:
+        return self.__name
+
+    @property
+    def members(self) -> list:
+        return self.__members
+
+    @property
+    def group_id(self) -> Optional[int]:
+        return self.__group_id
+
+    @staticmethod
+    def get(name_or_id: Union[str, int]) -> Optional["Group"]:
+        if isinstance(name_or_id, int):
+            return next((g for g in Group.ls() if g.group_id == name_or_id), None)
+        return Group(name_or_id)
+
+    @staticmethod
+    def add(name: str) -> Optional["Group"]:
+        Group.__client__.add(name)
+        return Group.get(name)
+
+    @staticmethod
+    def ls() -> list:
+        groups = Group.__client__.ls()
+        return [Group.get(group["name"]) for group in groups]
+
+    @staticmethod
+    def remove(name: str) -> None:
+        Group.__client__.rm(name)
+        return None
+
+    @staticmethod
+    def exists(name: str) -> bool:
+        return Group.__client__.exists(name)
+
+    def add_users(self, usernames: list) -> None:
+        if not self.name:
+            raise ValueError("Name is not set. Cannot add users.")
+        Group.__client__.add_users(self.name, ",".join(usernames))
+        self.__refresh__()
+
+    def remove_users(self, usernames: list) -> None:
+        if not self.name:
+            raise ValueError("Name is not set. Cannot remove users.")
+        Group.__client__.rm_users(self.name, ",".join(usernames))
+        self.__refresh__()
+
+    def is_member(self, user) -> bool:
+        if not self.name:
+            raise ValueError("Name is not set. Cannot check membership.")
+        return (
+            user.user_id in self.members
+            if hasattr(user, "user_id")
+            else user in self.members
+        )

datatailr/logging.py

@@ -0,0 +1,93 @@
+# *************************************************************************
+#
+#  Copyright (c) 2025 - Datatailr Inc.
+#  All Rights Reserved.
+#
+#  This file is part of Datatailr and subject to the terms and conditions
+#  defined in 'LICENSE.txt'. Unauthorized copying and/or distribution
+#  of this file, in parts or full, via any medium is strictly prohibited.
+# *************************************************************************
+
+import logging
+import os
+from logging import StreamHandler
+from logging.handlers import RotatingFileHandler
+from typing import Optional
+
+
+def get_log_level() -> int:
+    log_level = os.getenv("DATATAILR_LOG_LEVEL", "INFO").upper()
+    if log_level in ["TRACE", "DEBUG"]:
+        return logging.DEBUG
+    elif log_level == "INFO":
+        return logging.INFO
+    elif log_level == "WARNING":
+        return logging.WARNING
+    elif log_level == "ERROR":
+        return logging.ERROR
+    elif log_level == "CRITICAL":
+        return logging.CRITICAL
+    else:
+        return logging.INFO
+
+
+class DatatailrLogger:
+    def __init__(
+        self,
+        name: str,
+        log_file: Optional[str] = None,
+        log_level: int = get_log_level(),
+    ):
+        """
+        Initialize the DatatailrLogger.
+
+        :param name: Name of the logger.
+        :param log_file: Optional file to log messages to.
+        :param log_level: Logging level (default: logging.INFO).
+        """
+        self.logger = logging.getLogger(name)
+        self.logger.setLevel(log_level)
+
+        # Stream handler for stdout/stderr
+        stream_handler = StreamHandler()
+        stream_handler.setLevel(log_level)
+        stream_formatter = logging.Formatter(
+            "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+        )
+        stream_handler.setFormatter(stream_formatter)
+        self.logger.addHandler(stream_handler)
+
+        # Optional file handler
+        if log_file:
+            file_handler = RotatingFileHandler(
+                log_file, maxBytes=10 * 1024 * 1024, backupCount=5
+            )
+            file_handler.setLevel(log_level)
+            file_formatter = logging.Formatter(
+                "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+            )
+            file_handler.setFormatter(file_formatter)
+            self.logger.addHandler(file_handler)
+        self.enable_opentelemetry()
+
+    def get_logger(self):
+        """
+        Get the configured logger instance.
+
+        :return: Configured logger.
+        """
+        return self.logger
+
+    def enable_opentelemetry(self):
+        """
+        Enable OpenTelemetry integration if available.
+        """
+        try:
+            from opentelemetry.instrumentation.logging import LoggingInstrumentor  # type: ignore
+
+            LoggingInstrumentor().instrument(set_logging_format=True)
+            self.logger.debug("OpenTelemetry logging instrumentation enabled.")
+        except ImportError:
+            self.logger.debug(
+                "OpenTelemetry is not installed. Proceeding without instrumentation."
+            )

datatailr/sbin/run_job.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+
+# *************************************************************************
+#
+#  Copyright (c) 2025 - Datatailr Inc.
+#  All Rights Reserved.
+#
+#  This file is part of Datatailr and subject to the terms and conditions
+#  defined in 'LICENSE.txt'. Unauthorized copying and/or distribution
+#  of this file, in parts or full, via any medium is strictly prohibited.
+# *************************************************************************
+
+import importlib
+import os
+import pickle
+
+from datatailr import dt__Blob
+from datatailr.logging import DatatailrLogger
+
+logger = DatatailrLogger(os.path.abspath(__file__)).get_logger()
+
+
+def main():
+    entry_point = os.environ.get("DATATAILR_BATCH_ENTRYPOINT")
+    batch_run_id = os.environ.get("DATATAILR_BATCH_RUN_ID")
+    batch_id = os.environ.get("DATATAILR_BATCH_ID")
+    job_id = os.environ.get("DATATAILR_JOB_ID")
+
+    if entry_point is None:
+        raise ValueError(
+            "Environment variable 'DATATAILR_BATCH_ENTRYPOINT' is not set."
+        )
+    if batch_run_id is None:
+        raise ValueError("Environment variable 'DATATAILR_BATCH_RUN_ID' is not set.")
+    if batch_id is None:
+        raise ValueError("Environment variable 'DATATAILR_BATCH_ID' is not set.")
+    if job_id is None:
+        raise ValueError("Environment variable 'DATATAILR_JOB_ID' is not set.")
+
+    module_name, func_name = entry_point.split(":", 1)
+    module = importlib.import_module(module_name)
+    function = getattr(module, func_name)
+    if not callable(function):
+        raise ValueError(
+            f"The function '{func_name}' in module '{module_name}' is not callable."
+        )
+    result = function()
+    result_path = f"batch-results-{batch_run_id}-{job_id}.pkl"
+    with open(result_path, "wb") as f:
+        pickle.dump(result, f)
+    blob = dt__Blob()
+    blob.cp(result_path, "blob://")
+    logger.info(f"{result_path} copied to blob storage.")
+
+
+if __name__ == "__main__":
+    try:
+        logger.debug("Starting job execution...")
+        main()
+        logger.debug("Job executed successfully.")
+    except Exception as e:
+        logger.error(f"Error during job execution: {e}")
+        raise