blaxel 0.1.14rc54__tar.gz → 0.1.14rc56__tar.gz

{blaxel-0.1.14rc54 → blaxel-0.1.14rc56}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: blaxel
-Version: 0.1.14rc54
+Version: 0.1.14rc56
 Summary: Add your description here
 Author-email: cploujoux <cploujoux@blaxel.ai>
 License-File: LICENSE

blaxel-0.1.14rc56/integrationtest/bl_jobs.py

@@ -0,0 +1,12 @@
+import asyncio
+
+from blaxel.jobs import bl_job
+
+
+async def main():
+    job = bl_job("myjob")
+    print(await job.arun([{"name": "charlou", "age": 25}]))
+    print(job.run([{"name": "charlou", "age": 25}]))
+
+if __name__ == "__main__":
+    asyncio.run(main())

{blaxel-0.1.14rc54 → blaxel-0.1.14rc56}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "blaxel"
-version = "0.1.14.pre-54"
+version = "0.1.14.pre-56"
 description = "Add your description here"
 readme = "README.md"
 authors = [{ name = "cploujoux", email = "cploujoux@blaxel.ai" }]

blaxel-0.1.14rc56/src/blaxel/common/logger.py

@@ -0,0 +1,107 @@
+"""
+This module provides a custom colored formatter for logging and an initialization function
+to set up logging configurations for Blaxel applications.
+"""
+import json
+import logging
+import os
+from opentelemetry import trace
+
+class JsonFormatter(logging.Formatter):
+    """
+    A logger compatible with standard json logging.
+    """
+    def __init__(self):
+        super().__init__()
+        self.trace_id_name = os.environ.get('BL_LOGGER_TRACE_ID', 'trace_id')
+        self.span_id_name = os.environ.get('BL_LOGGER_SPAN_ID', 'span_id')
+        self.labels_name = os.environ.get('BL_LOGGER_LABELS', 'labels')
+        self.trace_id_prefix = os.environ.get('BL_LOGGER_TRACE_ID_PREFIX', '')
+        self.span_id_prefix = os.environ.get('BL_LOGGER_SPAN_ID_PREFIX', '')
+        self.task_index = os.environ.get('BL_TASK_KEY', 'TASK_INDEX')
+        self.task_prefix = os.environ.get('BL_TASK_PREFIX', '')
+        self.execution_key = os.environ.get('BL_EXECUTION_KEY', 'BL_EXECUTION_ID')
+        self.execution_prefix = os.environ.get('BL_EXECUTION_PREFIX', '')
+
+    def format(self, record):
+        """
+        Formats the log record by converting it to a JSON object with trace context and environment variables.
+        """
+        log_entry = {
+            'message': record.getMessage(),
+            'severity': record.levelname,
+            self.labels_name: {}
+        }
+
+        # Add trace context if available
+        current_span = trace.get_current_span()
+        if current_span.is_recording():
+            span_context = current_span.get_span_context()
+            log_entry[self.trace_id_name] = f"{self.trace_id_prefix}{span_context.trace_id}"
+            log_entry[self.span_id_name] = f"{self.span_id_prefix}{span_context.span_id}"
+
+        # Add task ID if available
+        task_id = os.environ.get(self.task_index)
+        if task_id:
+            log_entry[self.labels_name]['blaxel-task'] = f"{self.task_prefix}{task_id}"
+
+        # Add execution ID if available
+        execution_id = os.environ.get(self.execution_key)
+        if execution_id:
+            log_entry[self.labels_name]['blaxel-execution'] = f"{self.execution_prefix}{execution_id.split('-')[-1]}"
+
+        return json.dumps(log_entry)
+
+
+class ColoredFormatter(logging.Formatter):
+    """
+    A custom logging formatter that adds ANSI color codes to log levels for enhanced readability.
+
+    Attributes:
+        COLORS (dict): A mapping of log level names to their corresponding ANSI color codes.
+    """
+    COLORS = {
+        "DEBUG": "\033[1;36m",  # Cyan
+        "INFO": "\033[1;32m",  # Green
+        "WARNING": "\033[1;33m",  # Yellow
+        "ERROR": "\033[1;31m",  # Red
+        "CRITICAL": "\033[1;41m",  # Red background
+    }
+
+    def format(self, record):
+        """
+        Formats the log record by adding color codes based on the log level.
+
+        Parameters:
+            record (LogRecord): The log record to format.
+
+        Returns:
+            str: The formatted log message with appropriate color codes.
+        """
+        n_spaces = len("CRITICAL") - len(record.levelname)
+        tab = " " * n_spaces
+        color = self.COLORS.get(record.levelname, "\033[0m")
+        record.levelname = f"{color}{record.levelname}\033[0m:{tab}"
+        return super().format(record)
+
+def init_logger(log_level: str):
+    """
+    Initializes the logging configuration for Blaxel.
+
+    This function clears existing handlers for specific loggers, sets up a colored formatter,
+    and configures the root logger with the specified log level.
+
+    Parameters:
+        log_level (str): The logging level to set (e.g., "DEBUG", "INFO").
+    """
+    # Disable urllib3 logging
+    logging.getLogger('urllib3').setLevel(logging.CRITICAL)
+    logging.getLogger("httpx").setLevel(logging.CRITICAL)
+    handler = logging.StreamHandler()
+
+    logger_type = os.environ.get("BL_LOGGER", "http")
+    if logger_type == "json":
+        handler.setFormatter(JsonFormatter())
+    else:
+        handler.setFormatter(ColoredFormatter("%(levelname)s %(name)s - %(message)s"))
+    logging.basicConfig(level=log_level, handlers=[handler])

{blaxel-0.1.14rc54 → blaxel-0.1.14rc56}/src/blaxel/instrumentation/manager.py

@@ -7,6 +7,7 @@ import importlib
 import logging
 import signal
 import time
+import os
 from typing import Any, Dict, List, Optional, Type
 
 from opentelemetry import metrics, trace
@@ -164,14 +165,15 @@ class TelemetryManager:
         metrics.set_meter_provider(meter_provider)
         self.meter = meter_provider.get_meter(__name__)
 
-        # Set up the LoggerProvider
-        self.logger_provider = LoggerProvider(resource=resource)
-        set_logger_provider(self.logger_provider)
-        self.logger_provider.add_log_record_processor(
-            AsyncLogRecordProcessor(self.get_log_exporter())
-        )
-        handler = LoggingHandler(level=logging.NOTSET, logger_provider=self.logger_provider)
-        logging.getLogger().addHandler(handler)
+        logger_type = os.environ.get("BL_LOGGER", "http")
+        if logger_type == "http":
+            self.logger_provider = LoggerProvider(resource=resource)
+            set_logger_provider(self.logger_provider)
+            self.logger_provider.add_log_record_processor(
+                AsyncLogRecordProcessor(self.get_log_exporter())
+            )
+            handler = LoggingHandler(level=logging.NOTSET, logger_provider=self.logger_provider)
+            logging.getLogger().addHandler(handler)
 
         # Load and enable instrumentations
         for name, mapping in MAPPINGS.items():

blaxel-0.1.14rc56/src/blaxel/jobs/__init__.py

@@ -0,0 +1,193 @@
+import argparse
+import os
+import sys
+import asyncio
+import json
+import requests
+
+
+from typing import Any, Dict, Callable, Awaitable
+from logging import getLogger
+from ..client import client
+from ..common.env import env
+from ..common.internal import get_global_unique_hash
+from ..common.settings import settings
+from ..instrumentation.span import SpanManager
+
+
+
+class BlJobWrapper:
+    def get_arguments(self) -> Dict[str, Any]:
+        if not os.getenv('BL_EXECUTION_DATA_URL'):
+            parser = argparse.ArgumentParser()
+            # Parse known args, ignore unknown
+            args, unknown = parser.parse_known_args()
+            # Convert to dict and include unknown args
+            args_dict = vars(args)
+            # Add unknown args to dict
+            for i in range(0, len(unknown), 2):
+                if i + 1 < len(unknown):
+                    key = unknown[i].lstrip('-')
+                    args_dict[key] = unknown[i + 1]
+            return args_dict
+
+        response = requests.get(os.getenv('BL_EXECUTION_DATA_URL'))
+        data = response.json()
+        tasks = data.get('tasks', [])
+        return tasks[self.index] if self.index < len(tasks) else {}
+
+    @property
+    def index_key(self) -> str:
+        return os.getenv('BL_TASK_KEY', 'TASK_INDEX')
+
+    @property
+    def index(self) -> int:
+        index_value = os.getenv(self.index_key)
+        return int(index_value) if index_value else 0
+
+    def start(self, func: Callable):
+        """
+        Run a job defined in a function, it's run in the current process.
+        Handles both async and sync functions.
+        Arguments are passed as keyword arguments to the function.
+        """
+        attributes = {
+            "span.type": "job.start",
+        }
+        with SpanManager("blaxel-tracer").create_span("job.start", attributes) as span:
+            try:
+                parsed_args = self.get_arguments()
+                if asyncio.iscoroutinefunction(func):
+                    asyncio.run(func(**parsed_args))
+                else:
+                    func(**parsed_args)
+                span.set_attribute("job.start.success", True)
+                sys.exit(0)
+            except Exception as error:
+                span.set_attribute("job.start.error", str(error))
+                print('Job execution failed:', error, file=sys.stderr)
+                sys.exit(1)
+
+
+
+
+logger = getLogger(__name__)
+
+class BlJob:
+    def __init__(self, name: str):
+        self.name = name
+
+    @property
+    def internal_url(self):
+        """Get the internal URL for the job using a hash of workspace and job name."""
+        hash = get_global_unique_hash(settings.workspace, "job", self.name)
+        return f"{settings.run_internal_protocol}://bl-{settings.env}-{hash}.{settings.run_internal_hostname}"
+
+    @property
+    def forced_url(self):
+        """Get the forced URL from environment variables if set."""
+        env_var = self.name.replace("-", "_").upper()
+        if env[f"BL_JOB_{env_var}_URL"]:
+            return env[f"BL_JOB_{env_var}_URL"]
+        return None
+
+    @property
+    def external_url(self):
+        return f"{settings.run_url}/{settings.workspace}/jobs/{self.name}"
+
+    @property
+    def fallback_url(self):
+        if self.external_url != self.url:
+            return self.external_url
+        return None
+
+    @property
+    def url(self):
+        if self.forced_url:
+            return self.forced_url
+        if settings.run_internal_hostname:
+            return self.internal_url
+        return self.external_url
+
+    def call(self, url, input_data, headers: dict = {}, params: dict = {}):
+        body = {
+            "tasks": input_data
+        }
+
+        return client.get_httpx_client().post(
+            url+"/executions",
+            headers={
+                'Content-Type': 'application/json',
+                **headers
+            },
+            json=body,
+            params=params
+        )
+
+    async def acall(self, url, input_data, headers: dict = {}, params: dict = {}):
+        logger.debug(f"Job Calling: {self.name}")
+        body = {
+            "tasks": input_data
+        }
+
+        return await client.get_async_httpx_client().post(
+            url+"/executions",
+            headers={
+                'Content-Type': 'application/json',
+                **headers
+            },
+            json=body,
+            params=params
+        )
+
+    def run(self, input: Any, headers: dict = {}, params: dict = {}) -> str:
+        attributes = {
+            "job.name": self.name,
+            "span.type": "job.run",
+        }
+        with SpanManager("blaxel-tracer").create_active_span(self.name, attributes) as span:
+            logger.debug(f"Job Calling: {self.name}")
+            response = self.call(self.url, input, headers, params)
+            if response.status_code >= 400:
+                if not self.fallback_url:
+                    span.set_attribute("job.run.error", response.text)
+                    raise Exception(f"Job {self.name} returned status code {response.status_code} with body {response.text}")
+                response = self.call(self.fallback_url, input, headers, params)
+                if response.status_code >= 400:
+                    span.set_attribute("job.run.error", response.text)
+                    raise Exception(f"Job {self.name} returned status code {response.status_code} with body {response.text}")
+            span.set_attribute("job.run.result", response.text)
+            return response.text
+
+    async def arun(self, input: Any, headers: dict = {}, params: dict = {}) -> Awaitable[str]:
+        attributes = {
+            "job.name": self.name,
+            "span.type": "job.run",
+        }
+        with SpanManager("blaxel-tracer").create_active_span(self.name, attributes) as span:
+            logger.debug(f"Job Calling: {self.name}")
+            response = await self.acall(self.url, input, headers, params)
+            if response.status_code >= 400:
+                if not self.fallback_url:
+                    span.set_attribute("job.run.error", response.text)
+                    raise Exception(f"Job {self.name} returned status code {response.status_code} with body {response.text}")
+                response = await self.acall(self.fallback_url, input, headers, params)
+                if response.status_code >= 400:
+                    span.set_attribute("job.run.error", response.text)
+                    raise Exception(f"Job {self.name} returned status code {response.status_code} with body {response.text}")
+            span.set_attribute("job.run.result", response.text)
+            return response.text
+
+    def __str__(self):
+        return f"Job {self.name}"
+
+    def __repr__(self):
+        return self.__str__()
+
+
+
+def bl_job(name: str):
+    return BlJob(name)
+
+# Create a singleton instance
+bl_start_job = BlJobWrapper()

blaxel-0.1.14rc54/src/blaxel/common/logger.py

@@ -1,55 +0,0 @@
-"""
-This module provides a custom colored formatter for logging and an initialization function
-to set up logging configurations for Blaxel applications.
-"""
-
-import logging
-
-
-class ColoredFormatter(logging.Formatter):
-    """
-    A custom logging formatter that adds ANSI color codes to log levels for enhanced readability.
-
-    Attributes:
-        COLORS (dict): A mapping of log level names to their corresponding ANSI color codes.
-    """
-    COLORS = {
-        "DEBUG": "\033[1;36m",  # Cyan
-        "INFO": "\033[1;32m",  # Green
-        "WARNING": "\033[1;33m",  # Yellow
-        "ERROR": "\033[1;31m",  # Red
-        "CRITICAL": "\033[1;41m",  # Red background
-    }
-
-    def format(self, record):
-        """
-        Formats the log record by adding color codes based on the log level.
-
-        Parameters:
-            record (LogRecord): The log record to format.
-
-        Returns:
-            str: The formatted log message with appropriate color codes.
-        """
-        n_spaces = len("CRITICAL") - len(record.levelname)
-        tab = " " * n_spaces
-        color = self.COLORS.get(record.levelname, "\033[0m")
-        record.levelname = f"{color}{record.levelname}\033[0m:{tab}"
-        return super().format(record)
-
-def init_logger(log_level: str):
-    """
-    Initializes the logging configuration for Blaxel.
-
-    This function clears existing handlers for specific loggers, sets up a colored formatter,
-    and configures the root logger with the specified log level.
-
-    Parameters:
-        log_level (str): The logging level to set (e.g., "DEBUG", "INFO").
-    """
-    # Disable urllib3 logging
-    logging.getLogger('urllib3').setLevel(logging.CRITICAL)
-    logging.getLogger("httpx").setLevel(logging.CRITICAL)
-    handler = logging.StreamHandler()
-    handler.setFormatter(ColoredFormatter("%(levelname)s %(name)s - %(message)s"))
-    logging.basicConfig(level=log_level, handlers=[handler])

blaxel-0.1.14rc54/src/blaxel/jobs/__init__.py

@@ -1,55 +0,0 @@
-import argparse
-import os
-import sys
-import asyncio
-from typing import Any, Dict, Callable
-import requests
-
-class BlJob:
-    def get_arguments(self) -> Dict[str, Any]:
-        if not os.getenv('BL_EXECUTION_DATA_URL'):
-            parser = argparse.ArgumentParser()
-            # Parse known args, ignore unknown
-            args, unknown = parser.parse_known_args()
-            # Convert to dict and include unknown args
-            args_dict = vars(args)
-            # Add unknown args to dict
-            for i in range(0, len(unknown), 2):
-                if i + 1 < len(unknown):
-                    key = unknown[i].lstrip('-')
-                    args_dict[key] = unknown[i + 1]
-            return args_dict
-
-        response = requests.get(os.getenv('BL_EXECUTION_DATA_URL'))
-        data = response.json()
-        tasks = data.get('tasks', [])
-        return tasks[self.index] if self.index < len(tasks) else {}
-
-    @property
-    def index_key(self) -> str:
-        return os.getenv('BL_TASK_KEY', 'TASK_INDEX')
-
-    @property
-    def index(self) -> int:
-        index_value = os.getenv(self.index_key)
-        return int(index_value) if index_value else 0
-
-    def start(self, func: Callable):
-        """
-        Run a job defined in a function, it's run in the current process.
-        Handles both async and sync functions.
-        Arguments are passed as keyword arguments to the function.
-        """
-        try:
-            parsed_args = self.get_arguments()
-            if asyncio.iscoroutinefunction(func):
-                asyncio.run(func(**parsed_args))
-            else:
-                func(**parsed_args)
-            sys.exit(0)
-        except Exception as error:
-            print('Job execution failed:', error, file=sys.stderr)
-            sys.exit(1)
-
-# Create a singleton instance
-bl_job = BlJob()