fluidattacks-core 3.0.0__tar.gz → 4.0.0__tar.gz

{fluidattacks_core-3.0.0 → fluidattacks_core-4.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fluidattacks-core
-Version: 3.0.0
+Version: 4.0.0
 Summary: Fluid Attacks Core Library
 Author-email: Development <development@fluidattacks.com>
 License: MPL-2.0

{fluidattacks_core-3.0.0 → fluidattacks_core-4.0.0}/fluidattacks_core/logging/__init__.py

@@ -1,11 +1,21 @@
+from __future__ import annotations
+
 import logging
 import logging.config
 import sys
-from types import TracebackType
+from typing import TYPE_CHECKING
 
 from fluidattacks_core.logging.presets import DATE_FORMAT, PRODUCT_LOGGING
-from fluidattacks_core.logging.types import JobMetadata
-from fluidattacks_core.logging.utils import get_job_metadata, set_telemetry_metadata
+from fluidattacks_core.logging.sources.utils import (
+    set_commit_ref_name,
+    set_commit_sha,
+    set_product_id,
+)
+from fluidattacks_core.logging.utils import set_telemetry_metadata
+
+if TYPE_CHECKING:
+    from logging.config import _DictConfigArgs
+    from types import TracebackType
 
 
 def init_uncaught_exception_logging() -> None:
@@ -28,17 +38,18 @@ def init_uncaught_exception_logging() -> None:
     sys.excepthook = handle_uncaught_exception
 
 
-def init_logging() -> None:
-    logging.config.dictConfig(PRODUCT_LOGGING)
+def init_logging(preset: _DictConfigArgs | None = None) -> None:
+    logging.config.dictConfig(preset or PRODUCT_LOGGING)
     init_uncaught_exception_logging()
 
 
 __all__ = [
     "DATE_FORMAT",
     "PRODUCT_LOGGING",
-    "JobMetadata",
-    "get_job_metadata",
     "init_logging",
     "init_uncaught_exception_logging",
+    "set_commit_ref_name",
+    "set_commit_sha",
+    "set_product_id",
     "set_telemetry_metadata",
 ]

fluidattacks_core-4.0.0/fluidattacks_core/logging/filters.py

@@ -0,0 +1,13 @@
+import logging
+
+from fluidattacks_core.logging.sources.utils import get_environment
+
+
+class NoProductionFilter(logging.Filter):
+    def filter(self, _record: logging.LogRecord) -> bool:
+        return get_environment() != "production"
+
+
+class ProductionOnlyFilter(logging.Filter):
+    def filter(self, _record: logging.LogRecord) -> bool:
+        return get_environment() == "production"

{fluidattacks_core-3.0.0 → fluidattacks_core-4.0.0}/fluidattacks_core/logging/formatters.py

@@ -2,26 +2,24 @@ import logging
 import traceback
 from datetime import UTC, datetime
 from decimal import Decimal
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 import simplejson as json
 from pythonjsonlogger.json import JsonFormatter
 
+from fluidattacks_core.logging.sources import (
+    BatchSource,
+    ContainerSource,
+    DefaultSource,
+    LambdaSource,
+    PipelineSource,
+)
 from fluidattacks_core.logging.utils import (
-    get_environment_metadata,
-    get_job_metadata,
-    get_pipeline_environment,
-    get_pipeline_metadata,
     get_telemetry_metadata,
-    is_in_batch,
-    is_in_lambda,
 )
 
-# Main formats
-DATE_FORMAT = "%Y-%m-%dT%H:%M:%S%z"
-"""
-Default date format for logs.
-"""
+if TYPE_CHECKING:
+    from fluidattacks_core.logging.sources.types import SourceStrategy
 
 
 class ColorfulFormatter(logging.Formatter):
@@ -83,8 +81,7 @@ class CustomJsonFormatter(JsonFormatter):
         log_record["file_location"] = f"{record.filename}:{record.lineno}"
         log_record["lineno"] = record.lineno
 
-        self._add_service_fields(log_record)
-        self._add_deployment_fields(log_record)
+        self._add_source_fields(log_record)
 
         self._add_default_telemetry_fields(log_record)
         self._add_error_fields(log_record, record)
@@ -99,41 +96,27 @@ class CustomJsonFormatter(JsonFormatter):
                 return int(timestamp)
         return round(datetime.now(tz=UTC).timestamp() * 1000)
 
-    def _add_service_fields(self, log_record: dict[str, Any]) -> None:
-        """Add service information to the log record.
+    def _add_source_fields(self, log_record: dict[str, Any]) -> None:
+        """Add source information to the log record.
 
         It includes:
         - Source
         - Service
         - Version
+        - Deployment environment
+        - Other metadata according to the source
         """
-        batch_info = get_job_metadata().job_queue
-        source = (
-            f"batch/{batch_info}" if is_in_batch() else "lambda" if is_in_lambda() else "python"
-        )
-        service = get_environment_metadata().product_id
-        version = get_environment_metadata().version
-
-        log_record["ddsource"] = source
-        log_record["dd.service"] = service
-        log_record["dd.version"] = version
-
-    def _add_deployment_fields(self, log_record: dict[str, Any]) -> None:
-        """Add deployment information to the log record.
-
-        It includes:
-        - Environment
-        - Pipeline where the log was generated
-        """
-        env = get_environment_metadata().environment
-
-        log_record["dd.environment"] = env
-        log_record["deployment.environment"] = env
-
-        if pipeline := get_pipeline_environment():
-            log_record["deployment.pipeline.type"] = pipeline.upper()
-            for key, value in get_pipeline_metadata(pipeline).items():
-                log_record[f"deployment.pipeline.{key}"] = value
+        supported_sources: list[type[SourceStrategy]] = [
+            LambdaSource,
+            BatchSource,
+            PipelineSource,
+            ContainerSource,
+            DefaultSource,
+        ]
+        for source in supported_sources:
+            if source.detect():
+                log_record.update(source.log_metadata())
+                break
 
     def _add_default_telemetry_fields(self, log_record: dict[str, Any]) -> None:
         """Add default metadata fields to the log record.

{fluidattacks_core-3.0.0 → fluidattacks_core-4.0.0}/fluidattacks_core/logging/handlers.py

@@ -18,8 +18,8 @@ class DebuggingHandler(logging.StreamHandler[TextIO]):
     - Formatter: `ColorfulFormatter`
     """
 
-    def __init__(self) -> None:
-        super().__init__(sys.stderr)
+    def __init__(self, stream: TextIO = sys.stderr) -> None:
+        super().__init__(stream)
         self.addFilter(NoProductionFilter())
         self.setFormatter(ColorfulFormatter())
 
@@ -32,8 +32,8 @@ class ProductionSyncHandler(logging.StreamHandler[TextIO]):
     - Formatter: `CustomJsonFormatter`
     """
 
-    def __init__(self) -> None:
-        super().__init__(sys.stderr)
+    def __init__(self, stream: TextIO = sys.stderr) -> None:
+        super().__init__(stream)
         self.addFilter(ProductionOnlyFilter())
         self.setFormatter(CustomJsonFormatter())
 
@@ -46,8 +46,8 @@ class ProductionAsyncHandler(QueueHandler):
     - Formatter: `CustomJsonFormatter`
     """
 
-    def __init__(self) -> None:
-        handler = logging.StreamHandler(sys.stderr)
+    def __init__(self, stream: TextIO = sys.stderr) -> None:
+        handler = logging.StreamHandler(stream)
         handler.addFilter(ProductionOnlyFilter())
         handler.setFormatter(CustomJsonFormatter())
 

fluidattacks_core-4.0.0/fluidattacks_core/logging/presets.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from logging.config import _DictConfigArgs
+
+# Main formats
+DATE_FORMAT = "%Y-%m-%dT%H:%M:%S%z"
+"""
+Default date format for logs.
+"""
+
+
+PRODUCT_LOGGING: _DictConfigArgs = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "handlers": {
+        "production_handler": {
+            "class": "fluidattacks_core.logging.handlers.ProductionSyncHandler",
+            "stream": "ext://sys.stderr",
+        },
+        "debugging_handler": {
+            "class": "fluidattacks_core.logging.handlers.DebuggingHandler",
+            "stream": "ext://sys.stderr",
+        },
+    },
+    "root": {
+        "handlers": ["production_handler", "debugging_handler"],
+        "level": "INFO",
+    },
+}
+"""
+Default logging configuration dict for all the products.
+
+Required environment variables:
+- `PRODUCT_ID`
+- `CI_COMMIT_REF_NAME`
+- `CI_COMMIT_SHA`
+"""

fluidattacks_core-4.0.0/fluidattacks_core/logging/sources/__init__.py

@@ -0,0 +1,147 @@
+from fluidattacks_core.logging.sources.types import SourceStrategy
+from fluidattacks_core.logging.sources.utils import get_env_var, get_environment, get_version
+
+
+class DefaultSource(SourceStrategy):
+    @staticmethod
+    def detect() -> bool:
+        return True
+
+    @staticmethod
+    def log_metadata() -> dict[str, str]:
+        return {
+            "ddsource": "python",
+            "dd.service": get_env_var("PRODUCT_ID") or "unknown",
+            "dd.version": get_version(),
+            "deployment.environment": get_environment(),
+        }
+
+
+class LambdaSource(SourceStrategy):
+    @staticmethod
+    def detect() -> bool:
+        return get_env_var("AWS_LAMBDA_FUNCTION_NAME") is not None
+
+    @staticmethod
+    def log_metadata() -> dict[str, str]:
+        return {
+            "ddsource": "lambda",
+            "dd.service": get_env_var("AWS_LAMBDA_FUNCTION_NAME") or "unknown",
+            "dd.version": get_version(),
+            "deployment.environment": get_environment(),
+        }
+
+
+class BatchSource(SourceStrategy):
+    @staticmethod
+    def detect() -> bool:
+        return get_env_var("AWS_BATCH_JOB_ID") is not None
+
+    @staticmethod
+    def log_metadata() -> dict[str, str]:
+        job_name = get_env_var("JOB_DEFINITION_NAME")
+        job_queue = get_env_var("AWS_BATCH_JQ_NAME")
+        product_id = get_env_var("PRODUCT_ID")
+        service = job_name or (f"from-{job_queue}" if job_queue else product_id)
+        return {
+            "ddsource": "batch",
+            "dd.service": service or "unknown",
+            "dd.version": get_version(),
+            "deployment.environment": get_environment(),
+        }
+
+
+class PipelineSource(SourceStrategy):
+    @staticmethod
+    def detect() -> bool:
+        return PipelineSource._get_pipeline_metadata() is not None
+
+    @staticmethod
+    def log_metadata() -> dict[str, str]:
+        metadata = PipelineSource._get_pipeline_metadata()
+
+        return {
+            "ddsource": "ci",
+            "dd.service": get_env_var("PRODUCT_ID") or "unknown",
+            "dd.version": get_version(),
+            "deployment.environment": get_environment(),
+            **(metadata or {}),
+        }
+
+    @staticmethod
+    def _get_pipeline_metadata() -> dict[str, str] | None:
+        pipeline = None
+        if get_env_var("CI_JOB_ID"):
+            pipeline = "gitlab_ci"
+        elif get_env_var("CIRCLECI"):
+            pipeline = "circleci"
+        elif get_env_var("System.JobId"):
+            pipeline = "azure_devops"
+        elif get_env_var("BUILD_NUMBER"):
+            pipeline = "jenkins"
+
+        if pipeline is None:
+            return None
+
+        return {
+            "ddsource": f"ci/{pipeline}",
+            "deployment.pipeline.type": pipeline,
+            **(
+                {
+                    "deployment.pipeline.CI_JOB_ID": get_env_var("CI_JOB_ID") or "unknown",
+                    "deployment.pipeline.CI_JOB_URL": get_env_var("CI_JOB_URL") or "unknown",
+                }
+                if pipeline == "gitlab_ci"
+                else {}
+            ),
+            **(
+                {
+                    "deployment.pipeline.CIRCLE_BUILD_NUM": get_env_var("CIRCLE_BUILD_NUM")
+                    or "unknown",
+                    "deployment.pipeline.CIRCLE_BUILD_URL": get_env_var("CIRCLE_BUILD_URL")
+                    or "unknown",
+                }
+                if pipeline == "circleci"
+                else {}
+            ),
+            **(
+                {
+                    "deployment.pipeline.System.JobId": get_env_var("System.JobId") or "unknown",
+                }
+                if pipeline == "azure_devops"
+                else {}
+            ),
+            **(
+                {
+                    "deployment.pipeline.BUILD_NUMBER": get_env_var("BUILD_NUMBER") or "unknown",
+                    "deployment.pipeline.BUILD_ID": get_env_var("BUILD_ID") or "unknown",
+                    "deployment.pipeline.BUILD_URL": get_env_var("BUILD_URL") or "unknown",
+                }
+                if pipeline == "jenkins"
+                else {}
+            ),
+        }
+
+
+class ContainerSource(SourceStrategy):
+    @staticmethod
+    def detect() -> bool:
+        return get_env_var("CONTAINER_IMAGE") is not None
+
+    @staticmethod
+    def log_metadata() -> dict[str, str]:
+        return {
+            "ddsource": "container",
+            "dd.service": get_env_var("PRODUCT_ID") or "unknown",
+            "dd.version": get_version(),
+            "deployment.environment": get_environment(),
+        }
+
+
+__all__ = [
+    "BatchSource",
+    "ContainerSource",
+    "DefaultSource",
+    "LambdaSource",
+    "PipelineSource",
+]

fluidattacks_core-4.0.0/fluidattacks_core/logging/sources/types.py

@@ -0,0 +1,13 @@
+from abc import ABC, abstractmethod
+
+
+class SourceStrategy(ABC):
+    @staticmethod
+    @abstractmethod
+    def detect() -> bool:
+        """Detect if the current runtime is using this source."""
+
+    @staticmethod
+    @abstractmethod
+    def log_metadata() -> dict[str, str]:
+        """Get the metadata to be added to the log record."""

fluidattacks_core-4.0.0/fluidattacks_core/logging/sources/utils.py

@@ -0,0 +1,27 @@
+import os
+from typing import Literal
+
+
+def get_env_var(key: str) -> str | None:
+    return os.environ.get(key)
+
+
+def get_environment() -> Literal["development", "production"]:
+    branch = os.environ.get("CI_COMMIT_REF_NAME", "default")
+    return "production" if branch == "trunk" else "development"
+
+
+def get_version() -> str:
+    return os.environ.get("CI_COMMIT_SHA", "00000000")[:8]
+
+
+def set_product_id(product_id: str) -> None:
+    os.environ["PRODUCT_ID"] = product_id
+
+
+def set_commit_sha(commit_sha: str) -> None:
+    os.environ["CI_COMMIT_SHA"] = commit_sha
+
+
+def set_commit_ref_name(commit_ref_name: str) -> None:
+    os.environ["CI_COMMIT_REF_NAME"] = commit_ref_name

fluidattacks_core-4.0.0/fluidattacks_core/logging/utils.py

@@ -0,0 +1,37 @@
+import logging
+from typing import Any
+
+DEFAULT_TELEMETRY_METADATA: dict[str, Any] = {}
+
+
+def set_telemetry_metadata(config: dict[str, Any]) -> None:
+    DEFAULT_TELEMETRY_METADATA.update(config)
+
+
+def get_telemetry_metadata() -> dict[str, Any]:
+    return DEFAULT_TELEMETRY_METADATA
+
+
+def debug_logs() -> None:
+    """Test all the log levels in the root logger and a custom logger."""
+    root_logger = logging.getLogger()
+
+    root_logger.debug("This is a debug log")
+    root_logger.info("This is an info log")
+    root_logger.warning("This is a warning log")
+    root_logger.error("This is an error log")
+    root_logger.critical("This is a critical log")
+
+    logger = logging.getLogger("test-logger")
+    logger.debug("This is a debug log")
+    logger.info("This is an info log")
+    logger.warning("This is a warning log")
+    logger.error("This is an error log")
+    logger.critical("This is a critical log")
+
+    try:
+        msg = "Missing key"
+        raise KeyError(msg)  # noqa: TRY301
+    except KeyError as e:
+        root_logger.exception(e)  # noqa:TRY401
+        logger.exception(e)  # noqa:TRY401

{fluidattacks_core-3.0.0 → fluidattacks_core-4.0.0}/fluidattacks_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fluidattacks-core
-Version: 3.0.0
+Version: 4.0.0
 Summary: Fluid Attacks Core Library
 Author-email: Development <development@fluidattacks.com>
 License: MPL-2.0

{fluidattacks_core-3.0.0 → fluidattacks_core-4.0.0}/fluidattacks_core.egg-info/SOURCES.txt

@@ -43,8 +43,10 @@ fluidattacks_core/logging/filters.py
 fluidattacks_core/logging/formatters.py
 fluidattacks_core/logging/handlers.py
 fluidattacks_core/logging/presets.py
-fluidattacks_core/logging/types.py
 fluidattacks_core/logging/utils.py
+fluidattacks_core/logging/sources/__init__.py
+fluidattacks_core/logging/sources/types.py
+fluidattacks_core/logging/sources/utils.py
 fluidattacks_core/sarif/__init__.py
 fluidattacks_core/semver/__init__.py
 fluidattacks_core/semver/match_versions.py

{fluidattacks_core-3.0.0 → fluidattacks_core-4.0.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fluidattacks-core"
-version = "3.0.0"
+version = "4.0.0"
 description = "Fluid Attacks Core Library"
 authors = [{ name = "Development", email = "development@fluidattacks.com" }]
 license = { text = "MPL-2.0" }