datatailr 0.1.6__py3-none-any.whl → 0.1.10__py3-none-any.whl

datatailr/__init__.py

@@ -8,30 +8,12 @@
 #  of this file, in parts or full, via any medium is strictly prohibited.
 # *************************************************************************
 
-from datatailr.wrapper import (
-    dt__Blob,
-    dt__Dns,
-    dt__Email,
-    dt__Group,
-    dt__Job,
-    dt__Kv,
-    dt__Log,
-    dt__Node,
-    dt__Registry,
-    dt__Service,
-    dt__Settings,
-    dt__Sms,
-    dt__System,
-    dt__Tag,
-    dt__User,
-    mock_cli_tool,
-)
+from datatailr.wrapper import dt__System, mock_cli_tool
 from datatailr.group import Group
 from datatailr.user import User
 from datatailr.acl import ACL
 from datatailr.blob import Blob
 from datatailr.build import Image
-from datatailr.dt_json import dt_json
 from datatailr.utils import Environment, is_dt_installed
 from datatailr.version import __version__
 
@@ -50,21 +32,5 @@ __all__ = [
     "User",
     "__version__",
     "__provider__",
-    "dt__Blob",
-    "dt__Dns",
-    "dt__Email",
-    "dt__Group",
-    "dt__Job",
-    "dt__Kv",
-    "dt__Log",
-    "dt__Node",
-    "dt__Registry",
-    "dt__Service",
-    "dt__Settings",
-    "dt__Sms",
-    "dt__System",
-    "dt__Tag",
-    "dt__User",
-    "dt_json",
     "is_dt_installed",
 ]

datatailr/acl.py

@@ -8,6 +8,8 @@
 #  of this file, in parts or full, via any medium is strictly prohibited.
 # *************************************************************************
 
+from __future__ import annotations
+
 import json
 from typing import List, Optional, Union
 
@@ -40,9 +42,9 @@ class ACL:
         self.__user_can_write = False
         self.__world_readable = False
 
-        self.__parse_permissions()
+        self.__parse_permissions_string()
 
-    def __parse_permissions(self):
+    def __parse_permissions_string(self):
         """
         Parse the permissions and set the corresponding flags.
         """
@@ -59,6 +61,19 @@ class ACL:
         self.__world_readable = self.permissions[4] == "r"
         self.__world_writable = self.permissions[5] == "w"
 
+    def _set_permissions_string(self):
+        """
+        Set the permissions string based on the current flags.
+        """
+        self.permissions = (
+            f"{'r' if self.__user_can_read else '-'}"
+            f"{'w' if self.__user_can_write else '-'}"
+            f"{'r' if self.__group_can_read else '-'}"
+            f"{'w' if self.__group_can_write else '-'}"
+            f"{'r' if self.__world_readable else '-'}"
+            f"{'w' if self.__world_writable else '-'}"
+        )
+
     def __repr__(self):
         return (
             f"ACL(user={self.user}, group={self.group}, permissions={self.permissions})"
@@ -67,7 +82,7 @@ class ACL:
     def to_dict(self):
         return {
             "user": self.user.name,
-            "group": self.group.name if self.group else self.user.primary_group.name,
+            "group": self.group.name if self.group else "dtusers",
             "group_can_read": self.__group_can_read,
             "group_can_write": self.__group_can_write,
             "user_can_read": self.__user_can_read,
@@ -76,5 +91,22 @@ class ACL:
             "world_writable": self.__world_writable,
         }
 
+    @classmethod
+    def from_dict(cls, acl_dict: dict) -> ACL:
+        """
+        Create an ACL instance from a dictionary.
+        """
+        user = User(acl_dict["user"])
+        group = Group.get(acl_dict["group"]) if "group" in acl_dict else None
+        acl = cls(user=user, group=group)
+        acl.__group_can_read = acl_dict.get("group_can_read", False)
+        acl.__group_can_write = acl_dict.get("group_can_write", False)
+        acl.__user_can_read = acl_dict.get("user_can_read", False)
+        acl.__user_can_write = acl_dict.get("user_can_write", False)
+        acl.__world_readable = acl_dict.get("world_readable", False)
+        acl.__world_writable = acl_dict.get("world_writable", False)
+        acl._set_permissions_string()
+        return acl
+
     def to_json(self):
         return json.dumps(self.to_dict())

datatailr/blob.py

@@ -12,12 +12,13 @@ from __future__ import annotations
 
 import tempfile
 
-from datatailr import dt__Blob
+from datatailr.wrapper import dt__Blob
 
+# Datatailr Blob API Client
+__client__ = dt__Blob()
 
-class Blob:
-    __blob_storage__ = dt__Blob()
 
+class Blob:
     def ls(self, path: str) -> list[str]:
         """
         List files in the specified path.
@@ -25,7 +26,7 @@ class Blob:
         :param path: The path to list files from.
         :return: A list of file names in the specified path.
         """
-        return self.__blob_storage__.ls(path)
+        return __client__.ls(path)
 
     def get_file(self, name: str, path: str):
         """
@@ -35,7 +36,7 @@ class Blob:
             name (str): The name of the blob to retrieve.
             path (str): The path to store the blob as a file.
         """
-        return self.__blob_storage__.cp(f"blob:///{name}", path)
+        return __client__.cp(f"blob://{name}", path)
 
     def put_file(self, name: str, path: str):
         """
@@ -45,7 +46,7 @@ class Blob:
             name (str): The name of the blob to create.
             path (str): The path of the local file to copy.
         """
-        return self.__blob_storage__.cp(path, f"blob:///{name}")
+        return __client__.cp(path, f"blob://{name}")
 
     def exists(self, name: str) -> bool:
         """
@@ -57,7 +58,7 @@ class Blob:
         Returns:
             bool: True if the blob exists, False otherwise.
         """
-        return self.__blob_storage__.exists(name)
+        return __client__.exists(name)
 
     def delete(self, name: str):
         """
@@ -66,7 +67,7 @@ class Blob:
         Args:
             name (str): The name of the blob to delete.
         """
-        return self.__blob_storage__.rm(name)
+        return __client__.rm(name)
 
     def get_blob(self, name: str):
         """
@@ -81,9 +82,9 @@ class Blob:
         # Since direct reading and writting of blobs is not implemented yet, we are using a temporary file.
         # This is a workaround to allow reading the blob content directly from the blob storage.
 
-        with tempfile.NamedTemporaryFile(delete=False) as temp_file:
+        with tempfile.NamedTemporaryFile(delete=True) as temp_file:
             self.get_file(name, temp_file.name)
-            with open(temp_file.name, "rb") as f:
+            with open(temp_file.name, "r") as f:
                 return f.read()
 
     def put_blob(self, name: str, blob):
@@ -96,8 +97,7 @@ class Blob:
         """
         # Since direct reading and writting of blobs is not implemented yet, we are using a temporary file.
         # This is a workaround to allow writing the blob content directly to the blob storage.
-
-        with tempfile.NamedTemporaryFile(delete=False) as temp_file:
-            with open(temp_file.name, "wb") as f:
+        with tempfile.NamedTemporaryFile(delete=True) as temp_file:
+            with open(temp_file.name, "w") as f:
                 f.write(blob)
             self.put_file(name, temp_file.name)

datatailr/build/image.py

@@ -25,9 +25,13 @@ class Image:
     def __init__(
         self,
         acl: Optional[ACL] = None,
-        python_requirements: str = "",
+        python_requirements: str | list[str] = "",
         build_script_pre: str = "",
         build_script_post: str = "",
+        branch_name: Optional[str] = None,
+        commit_hash: Optional[str] = None,
+        path_to_repo: Optional[str] = None,
+        path_to_module: Optional[str] = None,
     ):
         if acl is None:
             signed_user = User.signed_user()
@@ -39,9 +43,11 @@ class Image:
             raise TypeError("acl must be an instance of ACL.")
         self.acl = acl or ACL(signed_user)
 
-        if os.path.isfile(python_requirements):
+        if isinstance(python_requirements, str) and os.path.isfile(python_requirements):
             with open(python_requirements, "r") as f:
                 python_requirements = f.read()
+        elif isinstance(python_requirements, list):
+            python_requirements = "\n".join(python_requirements)
         if not isinstance(python_requirements, str):
             raise TypeError(
                 "python_requirements must be a string or a file path to a requirements file."
@@ -65,10 +71,36 @@ class Image:
                 "build_script_post must be a string or a file path to a script file."
             )
         self.build_script_post = build_script_post
+        self.branch_name = branch_name
+        self.commit_hash = commit_hash
+        self.path_to_repo = path_to_repo
+        self.path_to_module = path_to_module
 
     def __repr__(self):
         return f"Image(acl={self.acl},)"
 
+    def update(self, **kwargs):
+        for key, value in kwargs.items():
+            if key == "acl" and not isinstance(value, ACL):
+                raise TypeError("acl must be an instance of ACL.")
+            elif key == "python_requirements" and not isinstance(value, str):
+                raise TypeError("python_requirements must be a string.")
+            elif key == "build_script_pre" and not isinstance(value, str):
+                raise TypeError("build_script_pre must be a string.")
+            elif key == "build_script_post" and not isinstance(value, str):
+                raise TypeError("build_script_post must be a string.")
+            elif (
+                key in ["branch_name", "commit_hash", "path_to_repo", "path_to_module"]
+                and value is not None
+                and not isinstance(value, str)
+            ):
+                raise TypeError(f"{key} must be a string or None.")
+            if key not in self.__dict__:
+                raise AttributeError(
+                    f"'{self.__class__.__name__}' object has no attribute '{key}'"
+                )
+            setattr(self, key, value)
+
     def to_dict(self):
         """
         Convert the Image instance to a dictionary representation.
@@ -78,6 +110,10 @@ class Image:
             "python_requirements": self.python_requirements,
             "build_script_pre": self.build_script_pre,
             "build_script_post": self.build_script_post,
+            "branch_name": self.branch_name,
+            "commit_hash": self.commit_hash,
+            "path_to_repo": self.path_to_repo,
+            "path_to_module": self.path_to_module,
         }
 
     def to_json(self):

datatailr/dt_json.py

@@ -10,6 +10,7 @@
 
 import importlib
 import json
+import base64
 
 
 class dt_json:
@@ -40,3 +41,34 @@ class dt_json:
     @classmethod
     def load(cls, json_file):
         return json.load(json_file)
+
+
+def encode_json(obj) -> str:
+    """
+    Encode an object to a JSON string with class information.
+    """
+    json_str = json.dumps(
+        obj, default=lambda o: o.__dict__ if hasattr(o, "__dict__") else o
+    )
+    return base64.b64encode(json_str.encode("utf-8")).decode("utf-8")
+
+
+def is_base64(s: str) -> bool:
+    try:
+        # Try decoding the string
+        base64.b64decode(s, validate=True)
+        return True
+    except (ValueError, TypeError):
+        # If decoding fails, it's not valid Base64
+        return False
+
+
+def decode_json(encoded_str: str):
+    """
+    Decode a JSON string back to an object.
+    """
+    if is_base64(encoded_str):
+        json_str = base64.b64decode(encoded_str.encode("utf-8")).decode("utf-8")
+    else:
+        json_str = encoded_str
+    return json.loads(json_str)

datatailr/errors.py

@@ -1,3 +1,14 @@
+##########################################################################
+#
+#  Copyright (c) 2025 - Datatailr Inc.
+#  All Rights Reserved.
+#
+#  This file is part of Datatailr and subject to the terms and conditions
+#  defined in 'LICENSE.txt'. Unauthorized copying and/or distribution
+#  of this file, in parts or full, via any medium is strictly prohibited.
+##########################################################################
+
+
 class DatatailrError(Exception):
     """Base class for all DataTailr exceptions."""
 
@@ -8,3 +19,9 @@ class BatchJobError(DatatailrError):
     """Exception raised for errors related to batch jobs."""
 
     pass
+
+
+class ScheduleError(DatatailrError):
+    """Exception raised for errors related to scheduling."""
+
+    pass

datatailr/group.py

@@ -10,17 +10,25 @@
 
 from typing import Optional, Union
 
-from datatailr import dt__Group, mock_cli_tool
+from datatailr.wrapper import dt__Group
+
+
+# Datatailr Group API Client
+__client__ = dt__Group()
 
 
 class Group:
-    """Representing a Datatailr Group
+    """
+    Representing a Datatailr Group.
+
     This class provides methods to interact with the Datatailr Group API.
     It allows you to create, update, delete, and manage groups within the Datatailr platform.
+
     Attributes:
         name (str): The name of the group.
         members (list): A list of members in the group.
         group_id (int): The unique identifier for the group.
+
     Static Methods:
         add(name: str) -> 'Group':
             Create a new group with the specified name.
@@ -32,6 +40,7 @@ class Group:
             Remove a group by its name.
         exists(name: str) -> bool:
             Check if a group exists by its name.
+
     Instance Methods:
         add_users(usernames: list) -> None:
             Add users to the group.
@@ -39,9 +48,6 @@ class Group:
             Remove users from the group.
     """
 
-    # Datatailr Group API Client
-    __client__ = dt__Group()
-
     def __init__(self, name: str):
         self.__name: str = name
         self.__members: list = []
@@ -69,9 +75,7 @@ class Group:
     def __refresh__(self):
         if not self.name:
             raise ValueError("Name is not set. Cannot refresh group.")
-        if isinstance(Group.__client__, mock_cli_tool):
-            return
-        group = Group.__client__.get(self.name)
+        group = __client__.get(self.name)
         if group:
             self.__name = group["name"]
             self.__members = group["members"]
@@ -99,33 +103,33 @@ class Group:
 
     @staticmethod
     def add(name: str) -> Optional["Group"]:
-        Group.__client__.add(name)
+        __client__.add(name)
         return Group.get(name)
 
     @staticmethod
     def ls() -> list:
-        groups = Group.__client__.ls()
+        groups = __client__.ls()
         return [Group.get(group["name"]) for group in groups]
 
     @staticmethod
     def remove(name: str) -> None:
-        Group.__client__.rm(name)
+        __client__.rm(name)
         return None
 
     @staticmethod
     def exists(name: str) -> bool:
-        return Group.__client__.exists(name)
+        return __client__.exists(name)
 
     def add_users(self, usernames: list) -> None:
         if not self.name:
             raise ValueError("Name is not set. Cannot add users.")
-        Group.__client__.add_users(self.name, ",".join(usernames))
+        __client__.add_users(self.name, ",".join(usernames))
         self.__refresh__()
 
     def remove_users(self, usernames: list) -> None:
         if not self.name:
             raise ValueError("Name is not set. Cannot remove users.")
-        Group.__client__.rm_users(self.name, ",".join(usernames))
+        __client__.rm_users(self.name, ",".join(usernames))
         self.__refresh__()
 
     def is_member(self, user) -> bool:

datatailr/logging.py

@@ -13,6 +13,8 @@ import os
 from logging import StreamHandler
 from logging.handlers import RotatingFileHandler
 from typing import Optional
+from datatailr import User
+from datatailr.wrapper import dt__Tag
 
 
 def get_log_level() -> int:
@@ -31,12 +33,28 @@ def get_log_level() -> int:
         return logging.INFO
 
 
+tag = dt__Tag()
+node_name = tag.get("node_name") or "local"
+node_ip = tag.get("node_ip")
+job_name = os.getenv("DATATAILR_JOB_NAME", "unknown_job")
+
+try:
+    user = User.signed_user().name
+except Exception:
+    import getpass
+
+    user = getpass.getuser()
+
+LOG_FORMAT = f"%(asctime)s - %(levelname)s - {node_name}:{node_ip} - {user} - {job_name} - %(name)s - [Line %(lineno)d]: %(message)s"
+
+
 class DatatailrLogger:
     def __init__(
         self,
         name: str,
         log_file: Optional[str] = None,
         log_level: int = get_log_level(),
+        log_format: str = LOG_FORMAT,
     ):
         """
         Initialize the DatatailrLogger.
@@ -51,9 +69,7 @@ class DatatailrLogger:
         # Stream handler for stdout/stderr
         stream_handler = StreamHandler()
         stream_handler.setLevel(log_level)
-        stream_formatter = logging.Formatter(
-            "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-        )
+        stream_formatter = logging.Formatter(log_format)
         stream_handler.setFormatter(stream_formatter)
         self.logger.addHandler(stream_handler)
 
@@ -63,9 +79,7 @@ class DatatailrLogger:
                 log_file, maxBytes=10 * 1024 * 1024, backupCount=5
             )
             file_handler.setLevel(log_level)
-            file_formatter = logging.Formatter(
-                "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-            )
+            file_formatter = logging.Formatter(log_format)
             file_handler.setFormatter(file_formatter)
             self.logger.addHandler(file_handler)
         self.enable_opentelemetry()
@@ -86,8 +100,5 @@ class DatatailrLogger:
             from opentelemetry.instrumentation.logging import LoggingInstrumentor  # type: ignore
 
             LoggingInstrumentor().instrument(set_logging_format=True)
-            self.logger.debug("OpenTelemetry logging instrumentation enabled.")
         except ImportError:
-            self.logger.debug(
-                "OpenTelemetry is not installed. Proceeding without instrumentation."
-            )
+            pass  # OpenTelemetry is not installed, skip instrumentation

datatailr/sbin/datatailr_run.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+
+# *************************************************************************
+#
+#  Copyright (c) 2025 - Datatailr Inc.
+#  All Rights Reserved.
+#
+#  This file is part of Datatailr and subject to the terms and conditions
+#  defined in 'LICENSE.txt'. Unauthorized copying and/or distribution
+#  of this file, in parts or full, via any medium is strictly prohibited.
+# *************************************************************************
+
+
+# The purpose of this script is to be the entrypoint for all jobs running on datatailr.
+# The main functions of the script are:
+#     1. Create a linux user and group for the job.
+#     2. Set the environment variables for the job.
+#     3. Run the job in a separate process, as the newly created user and pass all relevant environment variables.
+# There are muliple environment variables which are required for the job to run.
+# Some of them are necessary for the setup stage, which is executed directly in this script as the linux root user.
+# Others are passed to the job script, which is executed in a separate process with only the users' privileges and not as a root user.
+#
+# Setup environment variables:
+#     DATATAILR_USER - the user under which the job will run.
+#     DATATAILR_GROUP - the group under which the job will run.
+#     DATATAILR_UID - the user ID of the user as it is defined in the system.
+#     DATATAILR_GID - the group ID of the group as it is defined in the system.
+#     DATATAILR_JOB_TYPE - the type of job to run. (batch\service\app\excel\IDE)
+# Job environment variables (not all are always relevant, depending on the job type):
+#     DATATAILR_JOB_ARGUMENT_MAPPING - a JSON string mapping job argument names to their
+#     DATATAILR_BATCH_RUN_ID - the unique identifier for the batch run.
+#     DATATAILR_BATCH_ID - the unique identifier for the batch.
+#     DATATAILR_JOB_ID - the unique identifier for the job.
+
+
+import os
+from typing import Tuple
+from datatailr.logging import DatatailrLogger
+from datatailr.dt_json import encode_json
+
+
+logger = DatatailrLogger(os.path.abspath(__file__)).get_logger()
+
+
+def get_env_var(name: str, default: str | None = None) -> str:
+    """
+    Get an environment variable.
+    If the variable is not set, raise an error.
+    """
+    if name not in os.environ:
+        if default is not None:
+            return default
+        logger.error(f"Environment variable '{name}' is not set.")
+        raise ValueError(f"Environment variable '{name}' is not set.")
+    return os.environ[name]
+
+
+def create_user_and_group() -> Tuple[str, str]:
+    """
+    Create a user and group for the job.
+    The user and group names are taken from the environment variables DATATAILR_USER and DATATAILR_GROUP.
+    The group and user are created with the same uid and gid as passed in the environment variables DATATAILR_UID and DATATAILR_GID.
+    If the user or group already exists, do nothing.
+    """
+    user = get_env_var("DATATAILR_USER")
+    group = get_env_var("DATATAILR_GROUP")
+    uid = get_env_var("DATATAILR_UID")
+    gid = get_env_var("DATATAILR_GID")
+
+    # Create group if it does not exist
+    os.system(f"getent group {group} || groupadd {group} -g {gid} -o")
+
+    # Create user if it does not exist
+    os.system(
+        f"getent passwd {user} || useradd -g {group} -s /bin/bash -m {user} -u {uid} -o"
+    )
+    return user, group
+
+
+def run_command_as_user(command: str, user: str, env_vars: dict):
+    """
+    Run a command as a specific user with the given environment variables.
+    """
+    env_vars.update({"PATH": get_env_var("PATH")})
+    env_vars_str = " ".join(f"{key}='{value}'" for key, value in env_vars.items())
+    full_command = f"sudo -u {user} {env_vars_str} {command}"
+    logger.debug(f"Running command: {full_command}")
+    os.system(full_command)
+
+
+def main():
+    user, _ = create_user_and_group()
+    job_type = get_env_var("DATATAILR_JOB_TYPE")
+
+    job_name = get_env_var("DATATAILR_JOB_NAME")
+    job_id = get_env_var("DATATAILR_JOB_ID")
+    entrypoint = get_env_var("DATATAILR_ENTRYPOINT")
+
+    if job_type == "batch":
+        run_id = get_env_var("DATATAILR_BATCH_RUN_ID")
+        batch_id = get_env_var("DATATAILR_BATCH_ID")
+        job_argument_mapping = get_env_var(
+            "DATATAILR_JOB_ARGUMENT_MAPPING", encode_json({})
+        )
+        env = {
+            "DATATAILR_BATCH_RUN_ID": run_id,
+            "DATATAILR_BATCH_ID": batch_id,
+            "DATATAILR_JOB_ID": job_id,
+            "DATATAILR_BATCH_ENTRYPOINT": entrypoint,
+            "DATATAILR_JOB_ARGUMENT_MAPPING": job_argument_mapping,
+        }
+        run_command_as_user("datatailr_run_batch", user, env)
+    elif job_type == "service":
+        env = {
+            "DATATAILR_JOB_NAME": job_name,
+            "DATATAILR_JOB_ID": job_id,
+            "DATATAILR_ENTRYPOINT": entrypoint,
+        }
+        run_command_as_user("datatailr_run_service", user, env)
+    elif job_type == "app":
+        env = {
+            "DATATAILR_JOB_NAME": job_name,
+            "DATATAILR_JOB_ID": job_id,
+            "DATATAILR_ENTRYPOINT": entrypoint,
+        }
+        run_command_as_user("datatailr_run_app", user, env)
+    elif job_type == "excel":
+        env = {
+            "DATATAILR_JOB_NAME": job_name,
+            "DATATAILR_JOB_ID": job_id,
+            "DATATAILR_ENTRYPOINT": entrypoint,
+        }
+        run_command_as_user("datatailr_run_excel", user, env)
+    elif job_type == "IDE":
+        pass
+    else:
+        raise ValueError(f"Unknown job type: {job_type}")
+
+
+if __name__ == "__main__":
+    try:
+        logger.debug("Starting job execution...")
+        main()
+        logger.debug("Job executed successfully.")
+    except Exception as e:
+        logger.error(f"Error during job execution: {e}")
+        raise