datashare-python 0.7.3__tar.gz → 0.8.2__tar.gz

{datashare_python-0.7.3 → datashare_python-0.8.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datashare-python
-Version: 0.7.3
+Version: 0.8.2
 Summary: Manage Python tasks and local resources in Datashare
 Project-URL: Homepage, https://icij.github.io/datashare-python/
 Project-URL: Documentation, https://icij.github.io/datashare-python/

{datashare_python-0.7.3 → datashare_python-0.8.2}/datashare_python/config.py

@@ -1,3 +1,4 @@
+from enum import StrEnum
 from pathlib import Path
 from typing import Literal
 
@@ -78,13 +79,21 @@ class TemporalClientConfig(BaseModel):
 LogLevel = Literal["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"]
 
 
+class LogFormat(StrEnum):
+    JSON = "json"
+    LOGFMT = "logfmt"
+    DEFAULT = "default"
+
+
 class LoggingConfig(BaseModel):
-    log_in_json: bool = False
+    format: LogFormat = LogFormat.DEFAULT
     loggers: dict[str, LogLevel]
 
 
 _DEFAULT_LOGGERS = {datashare_python.__name__: "INFO"}
-_DEFAULT_LOGGING_CONFIG = LoggingConfig(log_in_json=True, loggers=_DEFAULT_LOGGERS)
+_DEFAULT_LOGGING_CONFIG = LoggingConfig(
+    format=LogFormat.DEFAULT, loggers=_DEFAULT_LOGGERS
+)
 
 
 class WorkerConfig(ICIJSettings, BaseModel):

{datashare_python-0.7.3 → datashare_python-0.8.2}/datashare_python/conftest.py

@@ -13,6 +13,7 @@ from temporalio import workflow
 
 from datashare_python.config import (
     DatashareClientConfig,
+    LogFormat,
     LoggingConfig,
     TemporalClientConfig,
     WorkerConfig,
@@ -93,7 +94,7 @@ def event_loop(
 @pytest.fixture(scope="session")
 def test_worker_config() -> WorkerConfig:
     logging_config = LoggingConfig(
-        log_in_json=False,
+        format=LogFormat.DEFAULT,
         loggers={
             "datashare_python": "DEBUG",
             "icij_common": "DEBUG",

{datashare_python-0.7.3 → datashare_python-0.8.2}/datashare_python/dependencies.py

@@ -40,7 +40,7 @@ def set_loggers(
     worker_config: WorkerConfig, worker_id: str, loggers: dict[str, LogLevel]
 ) -> None:
     setup_worker_loggers(
-        loggers=loggers, worker_id=worker_id, in_json=worker_config.logging.log_in_json
+        loggers=loggers, worker_id=worker_id, log_format=worker_config.logging.format
     )
     logger.info("worker loggers ready to log 💬")
 

{datashare_python-0.7.3 → datashare_python-0.8.2}/datashare_python/discovery.py

@@ -70,9 +70,8 @@ def discover(
     deps = []
     if deps_name is not None:
         deps = discover_dependencies(deps_name)
-    for mandatory in _MANDATORY_DEPS:
-        if mandatory not in deps:
-            deps.append(mandatory)
+    missing = [m for m in _MANDATORY_DEPS if m not in deps]
+    deps = missing + deps
     if deps:
         n_deps = len(deps)
         discovered += "\n"

{datashare_python-0.7.3 → datashare_python-0.8.2}/datashare_python/logging_.py

@@ -1,20 +1,39 @@
 import logging
+import numbers
 import sys
 from copy import copy
+from typing import Any
 
+import orjson
 from icij_common.logging_utils import DATE_FMT, STREAM_HANDLER_FMT
-from pythonjsonlogger.core import RESERVED_ATTRS, BaseJsonFormatter
+from pythonjsonlogger.core import BaseJsonFormatter
 from pythonjsonlogger.orjson import OrjsonFormatter
 from temporalio import activity, workflow
 
-from .config import LogLevel
+from .config import LogFormat, LogLevel
 from .interceptors import get_trace_context
 
+_BASE_ATTRS = [
+    "asctime",
+    "exc_info",
+    "filename",
+    "funcName",
+    "levelname",
+    "levelno",
+    "lineno",
+    "module",
+    "msecs",
+    "message",
+    "msg",
+    "name",
+    "pathname",
+]
 _ACT_LOGGER_ATTRS = ["activity_type", "activity_id", "activity_run_id"]
 _WF_LOGGED_ATTRS = ["workflow_type", "workflow_id", "workflow_run_id"]
 _TRACE_CONTEXT_ATTRS = ["trace_id", "parent_id", "traceparent"]
+
 _LOGGED_ATTRIBUTES = (
-    copy(RESERVED_ATTRS)
+    copy(_BASE_ATTRS)
     + _WF_LOGGED_ATTRS
     + _ACT_LOGGER_ATTRS
     + _TRACE_CONTEXT_ATTRS
@@ -28,7 +47,7 @@ _STREAM_HANDLER_FMT_WITH_WORKER_ID = (
 
 
 def setup_worker_loggers(
-    loggers: dict[str, LogLevel], *, worker_id: str | None, in_json: bool
+    loggers: dict[str, LogLevel], *, worker_id: str | None, log_format: LogFormat
 ) -> None:
     worker_filter = WorkerFilter(worker_id)
     for logger_name, level_str in loggers.items():
@@ -36,7 +55,9 @@ def setup_worker_loggers(
         logger = logging.getLogger(logger_name)
         logger.setLevel(level)
         logger.handlers = []
-        for handler in _get_worker_handlers(level, worker_filter, in_json=in_json):
+        for handler in _get_worker_handlers(
+            level, worker_filter, log_format=log_format
+        ):
             logger.addHandler(handler)
 
 
@@ -50,8 +71,13 @@ class WorkerFilter(logging.Filter):
             record.worker_id = self.worker_id
         if workflow.in_workflow():
             wf_info = workflow.info()
+            wf_info = {
+                "workflow_run_id": wf_info.run_id,
+                "workflow_id": self.workflow_id,
+                "workflow_type": self.workflow_type,
+            }
             for attr in _WF_LOGGED_ATTRS:
-                setattr(record, attr, getattr(wf_info, attr))
+                setattr(record, attr, wf_info[attr])
         if activity.in_activity():
             act_info = activity.info()
             for attr in _ACT_LOGGER_ATTRS:
@@ -64,23 +90,50 @@ class WorkerFilter(logging.Filter):
 
 
 def _get_worker_handlers(
-    level: int, worker_filter: WorkerFilter, *, in_json: bool
+    level: int, worker_filter: WorkerFilter, *, log_format: LogFormat
 ) -> list[logging.Handler]:
     stream_handler = logging.StreamHandler(sys.stderr)
-    if in_json:
-        fmt = _json_formatter(datefmt=DATE_FMT)
-    else:
-        if worker_filter.worker_id is not None:
-            fmt = _STREAM_HANDLER_FMT_WITH_WORKER_ID
-        else:
-            fmt = STREAM_HANDLER_FMT
-        fmt = logging.Formatter(fmt, DATE_FMT)
+    match log_format:
+        case LogFormat.JSON:
+            fmt = _json_formatter(datefmt=DATE_FMT)
+        case LogFormat.LOGFMT:
+            fmt = LogFmtFormatter(datefmt=DATE_FMT)
+        case LogFormat.DEFAULT:
+            if worker_filter.worker_id is not None:
+                fmt = _STREAM_HANDLER_FMT_WITH_WORKER_ID
+            else:
+                fmt = STREAM_HANDLER_FMT
+            fmt = logging.Formatter(fmt, DATE_FMT)
+        case _:
+            raise NotImplementedError(f"invalid log format: {log_format}")
     stream_handler.setFormatter(fmt)
     stream_handler.setLevel(level)
     stream_handler.addFilter(worker_filter)
     return [stream_handler]
 
 
+class LogFmtFormatter(logging.Formatter):
+    def format(self, record: logging.LogRecord) -> str:
+        logged = dict()
+        if record.exc_info and not record.exc_text:
+            record.exc_text = self.formatException(record.exc_info)
+            logged["exc_info"] = record.exc_text
+        for k, v in record.__dict__.items():
+            if k in _LOGGED_ATTRIBUTES and k != "exc_info":
+                logged[k] = _encode_value(v)
+        return " ".join(f"{k}={v}" for k, v in sorted(logged.items()))
+
+
+def _encode_value(value: Any) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, numbers.Number):
+        return str(value)
+    return orjson.dumps(value).decode()
+
+
 def _json_formatter(datefmt: str) -> BaseJsonFormatter:
     fmt = OrjsonFormatter(  # let's keep logging as fast as possible
         _LOGGED_ATTRIBUTES, datefmt=datefmt

{datashare_python-0.7.3 → datashare_python-0.8.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "datashare-python"
-version = "0.7.3"
+version = "0.8.2"
 description = "Manage Python tasks and local resources in Datashare"
 authors = [
   { name = "Clément Doumouro", email = "cdoumouro@icij.org" },