fluidattacks-core 2.16.0__py3-none-any.whl → 3.0.0__py3-none-any.whl

fluidattacks_core/git/constants.py

@@ -0,0 +1 @@
+DEFAULT_DOWNLOAD_BUFFER_SIZE = 64 * 1024  # 64KB

fluidattacks_core/git/download_file.py

@@ -4,23 +4,37 @@ from pathlib import Path
 import aiofiles
 import aiohttp
 
+from .constants import DEFAULT_DOWNLOAD_BUFFER_SIZE
+
 LOGGER = logging.getLogger(__name__)
 
 
-async def download_file(url: str, destination_path: str) -> bool:
-    async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=3600)) as session:  # noqa: SIM117
+async def download_file(
+    *,
+    url: str,
+    destination_path: str,
+    download_buffer_size: int = DEFAULT_DOWNLOAD_BUFFER_SIZE,
+) -> bool:
+    timeout = aiohttp.ClientTimeout(total=60 * 60, connect=30)
+    async with aiohttp.ClientSession(timeout=timeout) as session:  # noqa: SIM117
         async with session.get(url) as response:
-            if response.status == 200:
-                async with aiofiles.open(destination_path, "wb") as file:
-                    while True:
-                        try:
-                            chunk = await response.content.read(1024 * 10)
-                        except TimeoutError:
-                            LOGGER.warning("Read timeout")
-                            return False
-                        if not chunk:
-                            break
-                        await file.write(chunk)
-                return Path(destination_path).exists()
+            if response.status != 200:
+                LOGGER.error(
+                    "Failed to download file: HTTP %s, for path %s",
+                    response.status,
+                    destination_path,
+                )
+                return False
+
+            async with aiofiles.open(destination_path, "wb") as file:
+                while True:
+                    try:
+                        chunk = await response.content.read(download_buffer_size)
+                    except TimeoutError:
+                        LOGGER.exception("Read timeout for path %s", destination_path)
+                        return False
+                    if not chunk:
+                        break
+                    await file.write(chunk)
 
-    return False
+            return Path(destination_path).exists()

fluidattacks_core/git/download_repo.py

@@ -9,6 +9,7 @@ from git import GitError
 from git.cmd import Git
 from git.repo import Repo
 
+from .constants import DEFAULT_DOWNLOAD_BUFFER_SIZE
 from .delete_files import delete_out_of_scope_files
 from .download_file import download_file
 
@@ -83,12 +84,18 @@ async def download_repo_from_s3(
     download_url: str,
     destination_path: Path,
     git_ignore: list[str] | None = None,
+    *,
+    download_buffer_size: int = DEFAULT_DOWNLOAD_BUFFER_SIZE,
 ) -> bool:
     destination_path.parent.mkdir(parents=True, exist_ok=True)
     with tempfile.TemporaryDirectory(prefix="fluidattacks_", ignore_cleanup_errors=True) as tmpdir:
         tmp_path = Path(tmpdir)
         file_path = tmp_path / "repo.tar.gz"
-        result = await download_file(download_url, str(file_path.absolute()))
+        result = await download_file(
+            url=download_url,
+            destination_path=str(file_path.absolute()),
+            download_buffer_size=download_buffer_size,
+        )
         if not result:
             LOGGER.error("Failed to download repository from %s", download_url)
             return False

fluidattacks_core/logging/__init__.py

@@ -1,12 +1,44 @@
-from fluidattacks_core.logging.presets import BATCH_LOGGING, DATE_FORMAT, PRODUCT_LOGGING
+import logging
+import logging.config
+import sys
+from types import TracebackType
+
+from fluidattacks_core.logging.presets import DATE_FORMAT, PRODUCT_LOGGING
 from fluidattacks_core.logging.types import JobMetadata
 from fluidattacks_core.logging.utils import get_job_metadata, set_telemetry_metadata
 
+
+def init_uncaught_exception_logging() -> None:
+    logger = logging.getLogger("unhandled")
+
+    def handle_uncaught_exception(
+        exception_type: type[BaseException],
+        msg: BaseException,
+        traceback: TracebackType | None,
+    ) -> None:
+        if issubclass(exception_type, KeyboardInterrupt):
+            sys.__excepthook__(exception_type, msg, traceback)
+            return
+
+        logger.critical(
+            "Uncaught exception",
+            exc_info=(exception_type, msg, traceback),
+        )
+
+    sys.excepthook = handle_uncaught_exception
+
+
+def init_logging() -> None:
+    logging.config.dictConfig(PRODUCT_LOGGING)
+    init_uncaught_exception_logging()
+
+
 __all__ = [
-    "BATCH_LOGGING",
     "DATE_FORMAT",
     "PRODUCT_LOGGING",
     "JobMetadata",
     "get_job_metadata",
+    "init_logging",
+    "init_uncaught_exception_logging",
     "set_telemetry_metadata",
 ]

fluidattacks_core/logging/filters.py

@@ -1,6 +1,6 @@
 import logging
 
-from fluidattacks_core.logging.utils import get_job_metadata, is_trunk_branch
+from fluidattacks_core.logging.utils import is_trunk_branch
 
 
 class NoProductionFilter(logging.Filter):
@@ -15,13 +15,3 @@ class ProductionOnlyFilter(logging.Filter):
 
     def filter(self, _record: logging.LogRecord) -> bool:
         return is_trunk_branch()
-
-
-class BatchOnlyFilter(logging.Filter):
-    def filter(self, _record: logging.LogRecord) -> bool:
-        return get_job_metadata().job_id is not None
-
-
-class NoBatchFilter(logging.Filter):
-    def filter(self, _record: logging.LogRecord) -> bool:
-        return get_job_metadata().job_id is None

fluidattacks_core/logging/formatters.py

@@ -13,6 +13,8 @@ from fluidattacks_core.logging.utils import (
     get_pipeline_environment,
     get_pipeline_metadata,
     get_telemetry_metadata,
+    is_in_batch,
+    is_in_lambda,
 )
 
 # Main formats
@@ -101,19 +103,20 @@ class CustomJsonFormatter(JsonFormatter):
         """Add service information to the log record.
 
         It includes:
+        - Source
+        - Service
         - Version
-        - Product name
         """
         batch_info = get_job_metadata().job_queue
-        is_in_batch = get_job_metadata().job_id is not None
-        product_name = get_environment_metadata().product_id
-        service = f"{product_name}" + (f"/{batch_info}" if is_in_batch else "")
+        source = (
+            f"batch/{batch_info}" if is_in_batch() else "lambda" if is_in_lambda() else "python"
+        )
+        service = get_environment_metadata().product_id
         version = get_environment_metadata().version
 
-        log_record["dd.version"] = version
-        log_record["service.version"] = version
+        log_record["ddsource"] = source
         log_record["dd.service"] = service
-        log_record["service.name"] = service
+        log_record["dd.version"] = version
 
     def _add_deployment_fields(self, log_record: dict[str, Any]) -> None:
         """Add deployment information to the log record.

fluidattacks_core/logging/presets.py

@@ -1,7 +1,4 @@
-from fluidattacks_core.logging.filters import BatchOnlyFilter, NoBatchFilter
-from fluidattacks_core.logging.formatters import ColorfulFormatter
 from fluidattacks_core.logging.handlers import DebuggingHandler, ProductionSyncHandler
-from fluidattacks_core.logging.utils import get_environment_metadata, get_job_metadata
 
 # Main formats
 DATE_FORMAT = "%Y-%m-%dT%H:%M:%S%z"
@@ -10,69 +7,6 @@ Default date format for logs.
 """
 
 
-# Configuration for logging in batch environments
-_JOB_METADATA = get_job_metadata()
-_ENVIRONMENT_METADATA = get_environment_metadata()
-
-BATCH_LOGGING = {
-    "version": 1,
-    "disable_existing_loggers": False,
-    "filters": {
-        "batch_only": {"()": BatchOnlyFilter},
-        "no_batch": {"()": NoBatchFilter},
-    },
-    "formatters": {
-        "one_line_format": {
-            "class": "logging.Formatter",
-            "format": (
-                "{asctime} {levelname} [{name}] [{filename}:{lineno}] "
-                "[trace_id=None span_id=None "
-                f"service.name=batch/{_JOB_METADATA.job_queue} "
-                f"service.version={_ENVIRONMENT_METADATA.version} "
-                f"deployment.environment={_ENVIRONMENT_METADATA.environment} "
-                "trace_sampled=False]"
-                " - {message}, extra=None"
-            ),
-            "datefmt": DATE_FORMAT,
-            "style": "{",
-        },
-        "simple_format": {
-            "class": "logging.Formatter",
-            "format": "{asctime} [{levelname}] [{name}] {message}",
-            "datefmt": DATE_FORMAT,
-            "style": "{",
-        },
-        "colorful_format": {
-            "()": ColorfulFormatter,
-            "datefmt": DATE_FORMAT,
-            "style": "{",
-        },
-    },
-    "handlers": {
-        "batch_handler": {
-            "class": "logging.StreamHandler",
-            "stream": "ext://sys.stdout",
-            "formatter": "one_line_format",
-            "filters": ["batch_only"],
-        },
-        "console_handler": {
-            "class": "logging.StreamHandler",
-            "stream": "ext://sys.stdout",
-            "formatter": "colorful_format",
-            "filters": ["no_batch"],
-        },
-    },
-    "root": {
-        "handlers": ["batch_handler", "console_handler"],
-        "level": "INFO",
-    },
-}
-"""
-Logging configuration dict for batch environments.
-
-Root logger will have two handlers for batch and non-batch environments.
-"""
-
 PRODUCT_LOGGING = {
     "version": 1,
     "disable_existing_loggers": False,

fluidattacks_core/logging/utils.py

@@ -10,10 +10,20 @@ DEFAULT_TELEMETRY_METADATA = {}
 
 
 def is_trunk_branch() -> bool:
-    """Check if code is using the trunk branch."""
+    """Check if the code is using the trunk branch."""
     return os.environ.get("CI_COMMIT_REF_NAME", "default") == "trunk"
 
 
+def is_in_batch() -> bool:
+    """Check if the code is running in a batch environment."""
+    return os.environ.get("AWS_BATCH_JOB_ID") is not None
+
+
+def is_in_lambda() -> bool:
+    """Check if the code is running in an AWS Lambda function."""
+    return os.environ.get("AWS_LAMBDA_FUNCTION_NAME") is not None
+
+
 def get_job_metadata() -> JobMetadata:
     """Get the job metadata for applications running in batch environments."""
     return JobMetadata(
@@ -26,7 +36,9 @@ def get_job_metadata() -> JobMetadata:
 def get_environment_metadata() -> EnvironmentMetadata:
     """Get the environment metadata for applications."""
     environment = "production" if is_trunk_branch() else "development"
-    product_id = os.environ.get("PRODUCT_ID", "universe")
+    product_id = (
+        os.environ.get("AWS_LAMBDA_FUNCTION_NAME") or os.environ.get("PRODUCT_ID") or "universe"
+    )
     commit_sha = os.environ.get("CI_COMMIT_SHA", "00000000")
     commit_short_sha = commit_sha[:8]
 

{fluidattacks_core-2.16.0.dist-info → fluidattacks_core-3.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fluidattacks-core
-Version: 2.16.0
+Version: 3.0.0
 Summary: Fluid Attacks Core Library
 Author-email: Development <development@fluidattacks.com>
 License: MPL-2.0

{fluidattacks_core-2.16.0.dist-info → fluidattacks_core-3.0.0.dist-info}/RECORD

@@ -18,9 +18,10 @@ fluidattacks_core/git/__init__.py,sha256=aLiDZd-Jl7axe4zVvIiDQP2RPBiRvAf1jqEAc33
 fluidattacks_core/git/classes.py,sha256=vgCVOUF6tqeW0lKtD9giCNFQtzRit44bnu5qOAx7qCI,579
 fluidattacks_core/git/clone.py,sha256=alvidqUITrtTkvv4Ur9djI4Ch37QdhVWtHupMmV1eMc,7571
 fluidattacks_core/git/codecommit_utils.py,sha256=Ec1Ymk9F1DTTyRTdqrni77UUktGQgQB_jSq5n3wWy7Q,3422
+fluidattacks_core/git/constants.py,sha256=dTFn5bLkJ-VG-954MVJVHXxa4UCnvaurSM7GY73BiWk,49
 fluidattacks_core/git/delete_files.py,sha256=_EfPFl61tRK4CyQHL2QtvqCQQkQ38RTXVP0Db_d6rWg,1189
-fluidattacks_core/git/download_file.py,sha256=wJwMCJh_BCLR51N4GZOg-Xryjd2x3oguJEithXDgL9s,922
-fluidattacks_core/git/download_repo.py,sha256=BifLqDK-3p41hif1h5jqzE_zivqyG8HXyT51UVqGHOY,3882
+fluidattacks_core/git/download_file.py,sha256=0W0jhUiA6V7LRbAlbwUU3LbQiI1-UPqno7A70s2eo8s,1296
+fluidattacks_core/git/download_repo.py,sha256=GiZT0-kgqLTAg7uqV09P6V0AXyyPrxCe5QN3Fhp2iaE,4114
 fluidattacks_core/git/https_utils.py,sha256=V2Z9ClFq9F3sUvTqc_h6uf2PRdEzD-6MuC9zZJHy7_0,7036
 fluidattacks_core/git/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fluidattacks_core/git/remote.py,sha256=cPuyBMHeGrzRkEjroB6zlRLMA-QH2gIyIkGJNyf8wZc,1255
@@ -30,20 +31,20 @@ fluidattacks_core/git/warp.py,sha256=yLotfzBFWSWzM_DzI-hbp4t3hWggKNNGqBr-0mYQhLU
 fluidattacks_core/http/__init__.py,sha256=3Zz90L6e3_z-M-8Bvk_53rv-CFhPThkRGXnxCiQrmaU,60
 fluidattacks_core/http/client.py,sha256=jIhtGU2cKi5GZbxHq8WJOPgnk0beScRtxlz9tBSaKuw,2454
 fluidattacks_core/http/validations.py,sha256=h10Hr906KJqda1rJJb8eOqk1Xyyz81lAJ1glXeae4kM,3766
-fluidattacks_core/logging/__init__.py,sha256=p0OeNSbypmff-vrY3jZRjPL5h2aLe6OrJucAtbjI10U,382
-fluidattacks_core/logging/filters.py,sha256=3dPFJg90pNvk17F58CD-Qgw_LoWrEZ4z6QZbjaGfHm4,808
-fluidattacks_core/logging/formatters.py,sha256=70Hmz2AyhChGdcHgGa4fIqVkg_Gjogf23HlN0stIGt0,6748
+fluidattacks_core/logging/__init__.py,sha256=WFH1_0ctw4zSKLjsInHUvlFguIQuJJ2dVDW7VHyXPfY,1173
+fluidattacks_core/logging/filters.py,sha256=OqAS-cf-eDN8rWtbFpEK2kNPVyFTL31lKPonKHZ3kVA,492
+fluidattacks_core/logging/formatters.py,sha256=iGr5WnUNKPO9kxFpP87TT7pz62_UJaNXkXecywrQ-3c,6723
 fluidattacks_core/logging/handlers.py,sha256=g6PB5L84oOk9nwLTzLL0eB9zFnfLJSM69Qb2Y3qPf9g,4084
-fluidattacks_core/logging/presets.py,sha256=NyLbTOY7oUkL8gHhxEWGNSbUNtVmSUa_tFfIFmDD4FI,2876
+fluidattacks_core/logging/presets.py,sha256=JoUkhALSRNJ7qYxkuh7wdvWe0dXwKRxDGb6N-ihKArg,664
 fluidattacks_core/logging/types.py,sha256=aAPGXCEOSCtjVF36rAfWixAhiY7w6E3WDfd_pNAmNRw,233
-fluidattacks_core/logging/utils.py,sha256=vTIqYOvxN98vf3gHWPX9MDDl2P_zFUcxNk6SVXE3Q98,3815
+fluidattacks_core/logging/utils.py,sha256=RhmX-aFR41XpphU0FLXCFcWzyWZocNEZ14qgVB8SwAM,4195
 fluidattacks_core/sarif/__init__.py,sha256=vZkbzafVeqRPEc_dzq6oevZuNp50NNyNGa_eS0oNXnc,101519
 fluidattacks_core/semver/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fluidattacks_core/semver/match_versions.py,sha256=3L3C0TIVH0AtDpISvk5HHBXFSbJh5V7AINgfKEXYnYI,10157
 fluidattacks_core/serializers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fluidattacks_core/serializers/snippet.py,sha256=e520pZHC-fsNuYVNY30A7TcSugvUlFL6xdr74j5aCDM,12780
 fluidattacks_core/serializers/syntax.py,sha256=DkRsdMyMNrL0pRfsOSVAx79K8F0AmjBk676_d_v7PjM,15908
-fluidattacks_core-2.16.0.dist-info/METADATA,sha256=YuQp0jScKoZvGle2Ah823jpPegFGGUjcnAD5wDZCSZs,3200
-fluidattacks_core-2.16.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-fluidattacks_core-2.16.0.dist-info/top_level.txt,sha256=m49ZyZ2zPQmDBxkSpjb20wr-ZbGVXdOMFBZrDiP5Lb8,18
-fluidattacks_core-2.16.0.dist-info/RECORD,,
+fluidattacks_core-3.0.0.dist-info/METADATA,sha256=lo9JNVnYwKzZkgZtod7Ih95ITDjx5AcdUkgGZt7WhJc,3199
+fluidattacks_core-3.0.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+fluidattacks_core-3.0.0.dist-info/top_level.txt,sha256=m49ZyZ2zPQmDBxkSpjb20wr-ZbGVXdOMFBZrDiP5Lb8,18
+fluidattacks_core-3.0.0.dist-info/RECORD,,