datashare-python 0.7.3__py3-none-any.whl → 0.8.0__py3-none-any.whl

datashare_python/config.py

@@ -1,3 +1,4 @@
+from enum import StrEnum
 from pathlib import Path
 from typing import Literal
 
@@ -78,13 +79,21 @@ class TemporalClientConfig(BaseModel):
 LogLevel = Literal["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"]
 
 
+class LogFormat(StrEnum):
+    JSON = "json"
+    LOGFMT = "logfmt"
+    DEFAULT = "default"
+
+
 class LoggingConfig(BaseModel):
-    log_in_json: bool = False
+    format: LogFormat = LogFormat.DEFAULT
     loggers: dict[str, LogLevel]
 
 
 _DEFAULT_LOGGERS = {datashare_python.__name__: "INFO"}
-_DEFAULT_LOGGING_CONFIG = LoggingConfig(log_in_json=True, loggers=_DEFAULT_LOGGERS)
+_DEFAULT_LOGGING_CONFIG = LoggingConfig(
+    format=LogFormat.DEFAULT, loggers=_DEFAULT_LOGGERS
+)
 
 
 class WorkerConfig(ICIJSettings, BaseModel):

datashare_python/conftest.py

@@ -13,6 +13,7 @@ from temporalio import workflow
 
 from datashare_python.config import (
     DatashareClientConfig,
+    LogFormat,
     LoggingConfig,
     TemporalClientConfig,
     WorkerConfig,
@@ -93,7 +94,7 @@ def event_loop(
 @pytest.fixture(scope="session")
 def test_worker_config() -> WorkerConfig:
     logging_config = LoggingConfig(
-        log_in_json=False,
+        format=LogFormat.DEFAULT,
         loggers={
             "datashare_python": "DEBUG",
             "icij_common": "DEBUG",

datashare_python/dependencies.py

@@ -40,7 +40,7 @@ def set_loggers(
     worker_config: WorkerConfig, worker_id: str, loggers: dict[str, LogLevel]
 ) -> None:
     setup_worker_loggers(
-        loggers=loggers, worker_id=worker_id, in_json=worker_config.logging.log_in_json
+        loggers=loggers, worker_id=worker_id, format=worker_config.logging.format
     )
     logger.info("worker loggers ready to log 💬")
 

datashare_python/discovery.py

@@ -70,9 +70,8 @@ def discover(
     deps = []
     if deps_name is not None:
         deps = discover_dependencies(deps_name)
-    for mandatory in _MANDATORY_DEPS:
-        if mandatory not in deps:
-            deps.append(mandatory)
+    missing = [m for m in _MANDATORY_DEPS if m not in deps]
+    deps = missing + deps
     if deps:
         n_deps = len(deps)
         discovered += "\n"

datashare_python/logging_.py

@@ -1,20 +1,39 @@
 import logging
+import numbers
 import sys
 from copy import copy
+from typing import Any
 
+import orjson
 from icij_common.logging_utils import DATE_FMT, STREAM_HANDLER_FMT
-from pythonjsonlogger.core import RESERVED_ATTRS, BaseJsonFormatter
+from pythonjsonlogger.core import BaseJsonFormatter
 from pythonjsonlogger.orjson import OrjsonFormatter
 from temporalio import activity, workflow
 
-from .config import LogLevel
+from .config import LogFormat, LogLevel
 from .interceptors import get_trace_context
 
+_BASE_ATTRS = [
+    "asctime",
+    "exc_info",
+    "filename",
+    "funcName",
+    "levelname",
+    "levelno",
+    "lineno",
+    "module",
+    "msecs",
+    "message",
+    "msg",
+    "name",
+    "pathname",
+]
 _ACT_LOGGER_ATTRS = ["activity_type", "activity_id", "activity_run_id"]
 _WF_LOGGED_ATTRS = ["workflow_type", "workflow_id", "workflow_run_id"]
 _TRACE_CONTEXT_ATTRS = ["trace_id", "parent_id", "traceparent"]
+
 _LOGGED_ATTRIBUTES = (
-    copy(RESERVED_ATTRS)
+    copy(_BASE_ATTRS)
     + _WF_LOGGED_ATTRS
     + _ACT_LOGGER_ATTRS
     + _TRACE_CONTEXT_ATTRS
@@ -28,7 +47,7 @@ _STREAM_HANDLER_FMT_WITH_WORKER_ID = (
 
 
 def setup_worker_loggers(
-    loggers: dict[str, LogLevel], *, worker_id: str | None, in_json: bool
+    loggers: dict[str, LogLevel], *, worker_id: str | None, format: LogFormat
 ) -> None:
     worker_filter = WorkerFilter(worker_id)
     for logger_name, level_str in loggers.items():
@@ -36,7 +55,7 @@ def setup_worker_loggers(
         logger = logging.getLogger(logger_name)
         logger.setLevel(level)
         logger.handlers = []
-        for handler in _get_worker_handlers(level, worker_filter, in_json=in_json):
+        for handler in _get_worker_handlers(level, worker_filter, format=format):
             logger.addHandler(handler)
 
 
@@ -64,23 +83,50 @@ class WorkerFilter(logging.Filter):
 
 
 def _get_worker_handlers(
-    level: int, worker_filter: WorkerFilter, *, in_json: bool
+    level: int, worker_filter: WorkerFilter, *, format: LogFormat
 ) -> list[logging.Handler]:
     stream_handler = logging.StreamHandler(sys.stderr)
-    if in_json:
-        fmt = _json_formatter(datefmt=DATE_FMT)
-    else:
-        if worker_filter.worker_id is not None:
-            fmt = _STREAM_HANDLER_FMT_WITH_WORKER_ID
-        else:
-            fmt = STREAM_HANDLER_FMT
-        fmt = logging.Formatter(fmt, DATE_FMT)
+    match format:
+        case LogFormat.JSON:
+            fmt = _json_formatter(datefmt=DATE_FMT)
+        case LogFormat.LOGFMT:
+            fmt = LogFmtFormatter(datefmt=DATE_FMT)
+        case LogFormat.DEFAULT:
+            if worker_filter.worker_id is not None:
+                fmt = _STREAM_HANDLER_FMT_WITH_WORKER_ID
+            else:
+                fmt = STREAM_HANDLER_FMT
+            fmt = logging.Formatter(fmt, DATE_FMT)
+        case _:
+            raise NotImplementedError(f"invalid log format: {format}")
     stream_handler.setFormatter(fmt)
     stream_handler.setLevel(level)
     stream_handler.addFilter(worker_filter)
     return [stream_handler]
 
 
+class LogFmtFormatter(logging.Formatter):
+    def format(self, record: logging.LogRecord) -> str:
+        logged = dict()
+        if record.exc_info and not record.exc_text:
+            record.exc_text = self.formatException(record.exc_info)
+            logged["exc_info"] = record.exc_text
+        for k, v in record.__dict__.items():
+            if k in _LOGGED_ATTRIBUTES and k != "exc_info":
+                logged[k] = _encode_value(v)
+        return " ".join(f"{k}={v}" for k, v in sorted(logged.items()))
+
+
+def _encode_value(value: Any) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, numbers.Number):
+        return str(value)
+    return orjson.dumps(value).decode()
+
+
 def _json_formatter(datefmt: str) -> BaseJsonFormatter:
     fmt = OrjsonFormatter(  # let's keep logging as fast as possible
         _LOGGED_ATTRIBUTES, datefmt=datefmt

{datashare_python-0.7.3.dist-info → datashare_python-0.8.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datashare-python
-Version: 0.7.3
+Version: 0.8.0
 Summary: Manage Python tasks and local resources in Datashare
 Project-URL: Homepage, https://icij.github.io/datashare-python/
 Project-URL: Documentation, https://icij.github.io/datashare-python/

{datashare_python-0.7.3.dist-info → datashare_python-0.8.0.dist-info}/RECORD

@@ -1,27 +1,27 @@
 datashare_python/.gitignore,sha256=e-SRgnvGGdsjRrqgKsTzALz6Obx8IYiOjr0yaAxT6v8,22
 datashare_python/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datashare_python/__main__.py,sha256=g-fvS46zl9umKmGrSpl-OG-8PSuZgjqvTCqjpsZtSps,101
-datashare_python/config.py,sha256=_Cx4EB1yHXXcLUtw1OBlMk2SKcJRwxqJwgRu6klbxNg,3994
-datashare_python/conftest.py,sha256=MrmQKFcUipm_qn-cHsLovZMwMMtVxyK0s1lmKEx54bc,8651
+datashare_python/config.py,sha256=kgrj--t8aH4TKQpcak6f8H5Zny13LwPwvFDJDQk5K10,4137
+datashare_python/conftest.py,sha256=SiAVEwK5odlDljwyQMDxHlD-WEIem7oR04wAEoNH6jA,8673
 datashare_python/constants.py,sha256=a8-ceZKBVMXydcoNQ35fSjFjxeJ7dt-N6eAvqtPpf9g,320
-datashare_python/dependencies.py,sha256=KJuAp6Dmv8DQuFnGjbWiHu7StzZj97eBPDyZ_RfCQRc,4141
-datashare_python/discovery.py,sha256=BPB_Ak6d1-vcf9vAQA63IRb2U8h83_mIIi8MbKbFzQ0,7020
+datashare_python/dependencies.py,sha256=ZqIT-LmMMsx3QWf5WSgIY4LsY9bJ6TFJtKQKrHHt2_s,4135
+datashare_python/discovery.py,sha256=d8fpY9AUSH7WP6600TTmm1Cjr1TKOYBoO50cN61dPdU,6999
 datashare_python/exceptions.py,sha256=bVHEAXxDPKfxeeMC0hJXEsrJkgsKO2ESAhxWU96GA4M,496
 datashare_python/interceptors.py,sha256=Pl7GodPO4KbfflmacpW-vOUgLazjlXSlDNENbpOUt1c,6725
-datashare_python/logging_.py,sha256=XUhZTtofbOqJi1gwytYpUVqvoGPhoz5p2orsXs2FaWs,2968
+datashare_python/logging_.py,sha256=uSTNr6Een9zXNisirPZxJQsbOXMIADLgIPJp370lXng,4252
 datashare_python/objects.py,sha256=pE0DGNNkl1etxz5ed7T-EaGo1o9TONjH2Lg9u1qdAWU,7571
 datashare_python/task_client.py,sha256=oTmP8bvZW0UyhLNMi1AV3XIAx7hrdbxNRss2Mw2azEc,8435
 datashare_python/template.py,sha256=RxKTYLXoS_EQ8Jc41JkBXppPdbCFqDWfP3BmC0gvB5o,4024
 datashare_python/types_.py,sha256=9Hk1XqpdXbM1TnEzwvJ5G9ABbaCZW9KgBTtiPBVn_7k,649
 datashare_python/utils.py,sha256=gX3_RJEJS0sAYBNVfBLoWJu7_hIANhylSAohzXVW-yQ,17982
-datashare_python/worker-template.tar.gz,sha256=bOqoF6xVJRyFQaRYHIXXre31WYdmEqDLGeiXRr4Inqg,287091
+datashare_python/worker-template.tar.gz,sha256=g28XncbDlgLr1LS4WviGWZCS4V2jnrIt6ZlaXDI2ahs,287088
 datashare_python/worker.py,sha256=czrN9Z0fPFX-6KHinX8Orx4vb9tpta2e7Qs6H0NiYyE,7534
 datashare_python/cli/__init__.py,sha256=9BPWtssDgsVfWMsZ1TtZCla0EC_kai4RHttr8oNLYOE,1401
 datashare_python/cli/project.py,sha256=w32Gy9AOL5B00uDT4in7YUCt2g68FnNbvwg2M3a8G6o,946
 datashare_python/cli/task.py,sha256=8mvKGS21bZ14BgZ0Uo-dfameljkaI2ZBha80ywCy-E8,5822
 datashare_python/cli/utils.py,sha256=p69CQb0zfixuyBkiZprhdMCc_NuYwXyAn6vC9H1UzAw,911
 datashare_python/cli/worker.py,sha256=I4KTpFIpXFowioFn72Rm6LBCYlY-Dhp4NBIPvtRgUXE,5283
-datashare_python-0.7.3.dist-info/METADATA,sha256=sZEiq4mFYgmvkYuJOO6KWePleBYWDex722rVR3FSm3I,921
-datashare_python-0.7.3.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
-datashare_python-0.7.3.dist-info/entry_points.txt,sha256=ILE7auxabHWiu3GC-AunWnzjhOI_SbZp7D4GqZHlLw4,68
-datashare_python-0.7.3.dist-info/RECORD,,
+datashare_python-0.8.0.dist-info/METADATA,sha256=FDdSE58wp8Z4wHXQF8FVngy279LWO4vRTj6wgfpzGxc,921
+datashare_python-0.8.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+datashare_python-0.8.0.dist-info/entry_points.txt,sha256=ILE7auxabHWiu3GC-AunWnzjhOI_SbZp7D4GqZHlLw4,68
+datashare_python-0.8.0.dist-info/RECORD,,