deltacat 1.1.16__py3-none-any.whl → 1.1.17__py3-none-any.whl

deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "1.1.16"
+__version__ = "1.1.17"
 
 
 __all__ = [

deltacat/logs.py

@@ -49,11 +49,13 @@ class JsonFormatter(logging.Formatter):
         fmt_dict: dict = None,
         time_format: str = "%Y-%m-%dT%H:%M:%S",
         msec_format: str = "%s.%03dZ",
+        context_kwargs: Optional[Dict[str, Any]] = None,
     ):
         self.fmt_dict = fmt_dict if fmt_dict is not None else {"message": "message"}
         self.default_time_format = time_format
         self.default_msec_format = msec_format
         self.datefmt = None
+        self.additional_context = context_kwargs or {}
         if ray.is_initialized():
             self.ray_runtime_ctx: RuntimeContext = ray.get_runtime_context()
             self.context = {}
@@ -117,6 +119,9 @@ class JsonFormatter(logging.Formatter):
 
             message_dict["ray_runtime_context"] = self.context
 
+        if self.additional_context:
+            message_dict["additional_context"] = self.additional_context
+
         return json.dumps(message_dict, default=str)
 
 
@@ -159,6 +164,7 @@ def _create_rotating_file_handler(
     max_bytes_per_log_file: int = DEFAULT_MAX_BYTES_PER_LOG,
     backup_count: int = DEFAULT_BACKUP_COUNT,
     logging_format: Union[str, dict] = DEFAULT_LOG_FORMAT,
+    context_kwargs: Dict[str, Any] = None,
 ) -> FileHandler:
 
     if type(logging_level) is str:
@@ -176,7 +182,9 @@ def _create_rotating_file_handler(
     if type(logging_format) is str:
         handler.setFormatter(logging.Formatter(logging_format))
     else:
-        handler.setFormatter(JsonFormatter(logging_format))
+        handler.setFormatter(
+            JsonFormatter(logging_format, context_kwargs=context_kwargs)
+        )
 
     handler.setLevel(logging_level)
     return handler
@@ -205,6 +213,7 @@ def _configure_logger(
     log_dir: str,
     log_base_file_name: str,
     debug_log_base_file_name: str,
+    context_kwargs: Dict[str, Any] = None,
 ) -> Union[Logger, LoggerAdapter]:
     # This maintains log level of rotating file handlers
     primary_log_level = log_level
@@ -212,13 +221,19 @@ def _configure_logger(
     if log_level <= logging.getLevelName("DEBUG"):
         if not _file_handler_exists(logger, log_dir, debug_log_base_file_name):
             handler = _create_rotating_file_handler(
-                log_dir, debug_log_base_file_name, "DEBUG"
+                log_dir,
+                debug_log_base_file_name,
+                "DEBUG",
+                context_kwargs=context_kwargs,
             )
             _add_logger_handler(logger, handler)
             primary_log_level = logging.getLevelName("INFO")
     if not _file_handler_exists(logger, log_dir, log_base_file_name):
         handler = _create_rotating_file_handler(
-            log_dir, log_base_file_name, primary_log_level
+            log_dir,
+            log_base_file_name,
+            primary_log_level,
+            context_kwargs=context_kwargs,
         )
         _add_logger_handler(logger, handler)
 
@@ -226,7 +241,9 @@ def _configure_logger(
 
 
 def configure_deltacat_logger(
-    logger: Logger, level: int = None
+    logger: Logger,
+    level: int = None,
+    context_kwargs: Dict[str, Any] = None,
 ) -> Union[Logger, LoggerAdapter]:
     if level is None:
         level = logging.getLevelName(DELTACAT_SYS_LOG_LEVEL)
@@ -237,11 +254,14 @@ def configure_deltacat_logger(
         DELTACAT_SYS_LOG_DIR,
         DELTACAT_SYS_INFO_LOG_BASE_FILE_NAME,
         DELTACAT_SYS_DEBUG_LOG_BASE_FILE_NAME,
+        context_kwargs,
     )
 
 
 def configure_application_logger(
-    logger: Logger, level: int = None
+    logger: Logger,
+    level: int = None,
+    context_kwargs: Dict[str, Any] = None,
 ) -> Union[Logger, LoggerAdapter]:
     if level is None:
         level = logging.getLevelName(DELTACAT_APP_LOG_LEVEL)
@@ -252,4 +272,5 @@ def configure_application_logger(
         DELTACAT_APP_LOG_DIR,
         DELTACAT_APP_INFO_LOG_BASE_FILE_NAME,
         DELTACAT_APP_DEBUG_LOG_BASE_FILE_NAME,
+        context_kwargs,
     )

deltacat/tests/test_logs.py

@@ -126,3 +126,67 @@ class TestJsonFormatter(unittest.TestCase):
             "We expect task ID to be present inside a remote task",
         )
         ray.shutdown()
+
+    def test_format_when_ray_initialized_with_context_kwargs(self):
+        ray.init(local_mode=True, ignore_reinit_error=True)
+
+        formatter = JsonFormatter(
+            {"message": "msg"}, context_kwargs={"custom_key": "custom_val"}
+        )
+
+        record = LogRecord(
+            level="INFO",
+            name="test",
+            pathname="test",
+            lineno=0,
+            message="test_message",
+            msg="test_message",
+            args=None,
+            exc_info=None,
+        )
+
+        result = formatter.format(record)
+        result = json.loads(result)
+
+        self.assertEqual("test_message", result["message"])
+        self.assertTrue(ray.is_initialized())
+        self.assertIn("ray_runtime_context", result)
+        self.assertIn("job_id", result["ray_runtime_context"])
+        self.assertIn("node_id", result["ray_runtime_context"])
+        self.assertIn("worker_id", result["ray_runtime_context"])
+        self.assertNotIn(
+            "task_id",
+            result["ray_runtime_context"],
+            "We expect task ID not be present outside a remote task",
+        )
+        self.assertEqual("custom_val", result["additional_context"]["custom_key"])
+        ray.shutdown()
+
+    def test_format_with_context_kwargs(self):
+        ray.shutdown()
+        formatter = JsonFormatter(
+            {"message": "msg"}, context_kwargs={"custom_key": "custom_val"}
+        )
+
+        record = LogRecord(
+            level="INFO",
+            name="test",
+            pathname="test",
+            lineno=0,
+            message="test_message",
+            msg="test_message",
+            args=None,
+            exc_info=None,
+        )
+
+        result = formatter.format(record)
+
+        self.assertEqual(
+            {
+                "message": "test_message",
+                "additional_context": {"custom_key": "custom_val"},
+            },
+            json.loads(result),
+        )
+        self.assertFalse(ray.is_initialized())
+        self.assertNotIn("ray_runtime_context", json.loads(result))

{deltacat-1.1.16.dist-info → deltacat-1.1.17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.16
+Version: 1.1.17
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team

{deltacat-1.1.16.dist-info → deltacat-1.1.17.dist-info}/RECORD

@@ -1,7 +1,7 @@
-deltacat/__init__.py,sha256=o3hcQ85nWUeDYkaXi3ADAbqwM_c5ajyxzlx-Z2jdKbI,1778
+deltacat/__init__.py,sha256=x9MIXX9uM_Gf7hvnQuf0dQwXgyXztDLmGSbM1F27e10,1778
 deltacat/constants.py,sha256=_6oRI-3yp5c8J1qKGQZrt89I9-ttT_gSSvVsJ0h8Duc,1939
 deltacat/exceptions.py,sha256=yWM4RXK7uRrQc1VgJv6Lv2UiNZWAx2wolLq7cBwjlkg,12770
-deltacat/logs.py,sha256=6g16VkEFidbaMjgenAjggE1r2l664drMVhreRs8B1IQ,8438
+deltacat/logs.py,sha256=_UAc_6GiQR3mxccys32Cp2CZOKOVZ9L-AkNUAlzepns,9091
 deltacat/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/aws/clients.py,sha256=4eQvpkV1PzFfxog7EriuglOGGwNFHR5hbGYpjsNNPxk,6949
 deltacat/aws/constants.py,sha256=1HnDXrSokW-G3YA3qKEiv7fZVntDs1uSk6a7On-VG5k,1223
@@ -138,7 +138,7 @@ deltacat/storage/model/transform.py,sha256=t4hg1dKua8VPeMFgyllkWdzq-L5M_DRG0HD9s
 deltacat/storage/model/types.py,sha256=hj7MmjjVmKT-R9sMUulOWG-FByGZKKaYXNnOWW32mP0,1608
 deltacat/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/tests/test_exceptions.py,sha256=V3jUQClHLD24tS18tnGvNIt0psn2WFT3Nf_CIvSqL08,3140
-deltacat/tests/test_logs.py,sha256=Bq0kfzOVW0UbJL4Hayqy1k1QDXLu5hlvQX_cOJdsgYs,3851
+deltacat/tests/test_logs.py,sha256=QmnArioPo0dbOUXjaG-b7HEg1GaYI8NDhinPDmzwjfQ,5889
 deltacat/tests/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/tests/aws/test_clients.py,sha256=23GMWfz27WWBDXSqphG9mfputsyS7j3I5P_HRk4YoKE,3790
 deltacat/tests/aws/test_s3u.py,sha256=FsYCH8K8DsDRPOtTp-w1Nu3ATqt4p1mqDo6aVJV-SbU,7918
@@ -220,8 +220,8 @@ deltacat/utils/ray_utils/concurrency.py,sha256=JDVwMiQWrmuSlyCWAoiq9ctoJ0XADEfDD
 deltacat/utils/ray_utils/dataset.py,sha256=waHdtH0c835a-2t51HYRHnulfC0_zBxx8mFSAPvPSPM,3274
 deltacat/utils/ray_utils/performance.py,sha256=d7JFM7vTXHzkGx9qNQcZzUWajnqINvYRwaM088_FpsE,464
 deltacat/utils/ray_utils/runtime.py,sha256=rB0A-tU9WZHz0J11LzJdANYtL397YyuemcA1l-K9dAw,5029
-deltacat-1.1.16.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-deltacat-1.1.16.dist-info/METADATA,sha256=EvOmjI60akKZ6BKEWQ2_KZJxr-Bp6wMZOU6-zV1EDos,1734
-deltacat-1.1.16.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-deltacat-1.1.16.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
-deltacat-1.1.16.dist-info/RECORD,,
+deltacat-1.1.17.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+deltacat-1.1.17.dist-info/METADATA,sha256=SxYHCGGCCaUP0Yej0eT0BMV3K8lQdOmodQj6VSnALcU,1734
+deltacat-1.1.17.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+deltacat-1.1.17.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
+deltacat-1.1.17.dist-info/RECORD,,