karton-core 5.6.0__py3-none-any.whl → 5.7.0__py3-none-any.whl

karton/core/__version__.py

@@ -1 +1 @@
-__version__ = "5.6.0"
+__version__ = "5.7.0"

karton/core/backend.py

@@ -1021,21 +1021,33 @@ class KartonBackend:
             self.s3.delete_objects(Bucket=bucket, Delete={"Objects": delete_objects})
 
     def remove_object_versions(
-        self, bucket: str, object_versions: Dict[str, List[str]]
+        self,
+        bucket: str,
+        object_versions: Dict[str, List[str]],
+        explicit_version_null: bool = False,
     ) -> None:
         """
         Bulk remove resource object versions from object storage
 
         :param bucket: Bucket name
         :param object_versions: Object version identifiers
+        :param explicit_version_null: |
+            Some S3 providers (e.g. MinIO) need a reference
+            to "null" version explicitly when versioning is in suspended state. On the
+            other hand, some providers refuse to delete "null" versions when bucket
+            versioning is disabled.
+            See also: https://github.com/CERT-Polska/karton/issues/273.
         """
-        versions = iter(
-            (uid, version_id)
-            for uid, versions in object_versions.items()
-            for version_id in versions
-        )
         deletion_chunks = chunks(
-            [{"Key": uid, "VersionId": version_id} for uid, version_id in versions],
+            [
+                (
+                    {"Key": uid, "VersionId": version_id}
+                    if version_id != "null" or explicit_version_null
+                    else {"Key": uid}
+                )
+                for uid, versions in object_versions.items()
+                for version_id in versions
+            ],
             100,
         )
         for delete_objects in deletion_chunks:
@@ -1060,17 +1072,20 @@ class KartonBackend:
                 raise e
         return False
 
-    def log_identity_output(self, identity: str, headers: Dict[str, Any]) -> None:
+    def log_identity_output(
+        self, identity: str, headers: Dict[str, Any], task_tracking_ttl: int
+    ) -> None:
         """
         Store the type of task outputted for given producer to
         be used in tracking karton service connections.
 
         :param identity: producer identity
         :param headers: outputted headers
+        :param task_tracking_ttl: expire time (in seconds)
         """
 
         self.redis.sadd(f"{KARTON_OUTPUTS_NAMESPACE}:{identity}", json.dumps(headers))
-        self.redis.expire(f"{KARTON_OUTPUTS_NAMESPACE}:{identity}", 60 * 60 * 24 * 30)
+        self.redis.expire(f"{KARTON_OUTPUTS_NAMESPACE}:{identity}", task_tracking_ttl)
 
     def get_outputs(self) -> List[KartonOutputs]:
         """

karton/core/base.py

@@ -36,6 +36,9 @@ class KartonBase(abc.ABC):
         backend: Optional[KartonBackend] = None,
     ) -> None:
         self.config = config or Config()
+        self.enable_publish_log = self.config.getboolean(
+            "logging", "enable_publish", True
+        )
 
         # If not passed via constructor - get it from class
         if identity is not None:
@@ -108,7 +111,7 @@ class KartonBase(abc.ABC):
         )
         logger.addHandler(stream_handler)
 
-        if not self.debug:
+        if not self.debug and self.enable_publish_log:
             logger.addHandler(self._log_handler)
 
     @property

karton/core/config.py

@@ -111,12 +111,10 @@ class Config(object):
         return True
 
     @overload
-    def getint(self, section_name: str, option_name: str, fallback: int) -> int:
-        ...
+    def getint(self, section_name: str, option_name: str, fallback: int) -> int: ...
 
     @overload
-    def getint(self, section_name: str, option_name: str) -> Optional[int]:
-        ...
+    def getint(self, section_name: str, option_name: str) -> Optional[int]: ...
 
     def getint(
         self, section_name: str, option_name: str, fallback: Optional[int] = None
@@ -131,12 +129,12 @@ class Config(object):
         return int(value)
 
     @overload
-    def getboolean(self, section_name: str, option_name: str, fallback: bool) -> bool:
-        ...
+    def getboolean(
+        self, section_name: str, option_name: str, fallback: bool
+    ) -> bool: ...
 
     @overload
-    def getboolean(self, section_name: str, option_name: str) -> Optional[bool]:
-        ...
+    def getboolean(self, section_name: str, option_name: str) -> Optional[bool]: ...
 
     def getboolean(
         self, section_name: str, option_name: str, fallback: Optional[bool] = None

karton/core/karton.py

@@ -1,6 +1,7 @@
 """
 Base library for karton subsystems.
 """
+
 import abc
 import argparse
 import sys

karton/core/resource.py

@@ -39,6 +39,7 @@ class ResourceBase(object):
         bucket: Optional[str] = None,
         metadata: Optional[Dict[str, Any]] = None,
         sha256: Optional[str] = None,
+        fd: Optional[IO[bytes]] = None,
         _uid: Optional[str] = None,
         _size: Optional[int] = None,
         _flags: Optional[List[str]] = None,
@@ -66,6 +67,16 @@ class ResourceBase(object):
                     for byte_block in iter(lambda: f.read(4096), b""):
                         sha256_hash.update(byte_block)
                 sha256 = sha256_hash.hexdigest()
+        elif fd is not None:
+            if calculate_hash:
+                # we need to calculate the whole hash and return pos as it was
+                sha256_hash = hashlib.sha256()
+                last_position = fd.tell()
+                fd.seek(0)
+                for byte_block in iter(lambda: fd.read(4096), b""):
+                    sha256_hash.update(byte_block)
+                sha256 = sha256_hash.hexdigest()
+                fd.seek(last_position)
         elif content:
             if isinstance(content, str):
                 self._content = content.encode()
@@ -190,6 +201,7 @@ class LocalResource(ResourceBase):
             bucket=bucket,
             metadata=metadata,
             sha256=sha256,
+            fd=fd,
             _uid=uid,
             _flags=_flags,
         )

karton/core/test.py

@@ -1,11 +1,12 @@
 """
 Test stubs for karton subsystem unit tests
 """
+
 import hashlib
 import logging
 import unittest
 from collections import defaultdict
-from typing import Any, BinaryIO, Dict, List, Union, cast
+from typing import Any, BinaryIO, Dict, List, Optional, Union, cast
 from unittest import mock
 
 from .backend import KartonBackend, KartonMetrics
@@ -66,7 +67,7 @@ class BackendMock:
         bucket: str,
         object_uid: str,
         content: Union[bytes, BinaryIO],
-        length: int = None,
+        length: Optional[int] = None,
     ) -> None:
         log.debug("Uploading object %s to bucket %s", object_uid, bucket)
         if isinstance(content, bytes):

karton/system/system.py

@@ -31,6 +31,7 @@ class SystemService(KartonServiceBase):
     TASK_DISPATCHED_TIMEOUT = 24 * 3600
     TASK_STARTED_TIMEOUT = 24 * 3600
     TASK_CRASHED_TIMEOUT = 3 * 24 * 3600
+    TASK_TRACKING_TTL = 30 * 24 * 3600
 
     def __init__(self, config: Optional[Config]) -> None:
         super().__init__(config=config)
@@ -49,6 +50,15 @@ class SystemService(KartonServiceBase):
         self.crash_started_tasks_on_timeout = self.config.getboolean(
             "system", "crash_started_tasks_on_timeout", False
         )
+        self.enable_null_version_deletion = self.config.getboolean(
+            "system", "enable_null_version_deletion", False
+        )
+        self.enable_task_tracking = self.config.getboolean(
+            "system", "enable_task_tracking", True
+        )
+        self.task_tracking_ttl = self.config.getint(
+            "system", "task_tracking_ttl", self.TASK_TRACKING_TTL
+        )
 
         self.last_gc_trigger = time.time()
 
@@ -61,6 +71,9 @@ class SystemService(KartonServiceBase):
             " task_crashed_timeout:\t%s\n"
             " enable_gc:\t%s\n"
             " enable_router:\t%s\n"
+            " enable_null_version_deletion:\t%s\n"
+            " enable_task_tracking:\t%s\n"
+            " task_tracking_ttl:\t%s\n"
             " crash_started_tasks_on_timeout:\t%s",
             self.gc_interval,
             self.task_dispatched_timeout,
@@ -68,6 +81,9 @@ class SystemService(KartonServiceBase):
             self.task_crashed_timeout,
             self.enable_gc,
             self.enable_router,
+            self.enable_null_version_deletion,
+            self.enable_task_tracking,
+            self.task_tracking_ttl,
             self.crash_started_tasks_on_timeout,
         )
 
@@ -88,7 +104,11 @@ class SystemService(KartonServiceBase):
                     del resources_to_remove[resource.uid]
         # Remove unreferenced resources
         if resources_to_remove:
-            self.backend.remove_object_versions(karton_bucket, resources_to_remove)
+            self.backend.remove_object_versions(
+                karton_bucket,
+                resources_to_remove,
+                explicit_version_null=self.enable_null_version_deletion,
+            )
 
     def gc_collect_tasks(self) -> None:
         self.log.debug("GC: gc_collect_tasks started")
@@ -218,10 +238,13 @@ class SystemService(KartonServiceBase):
     def route_task(self, task: Task, binds: List[KartonBind]) -> None:
         # Performs routing of task
         self.log.info("[%s] Processing task %s", task.root_uid, task.task_uid)
-        # store the producer-task relationship in redis for task tracking
-        self.backend.log_identity_output(
-            task.headers.get("origin", "unknown"), task.headers
-        )
+        # if enabled, store the producer-task relationship in redis for task tracking
+        if self.enable_task_tracking:
+            self.backend.log_identity_output(
+                task.headers.get("origin", "unknown"),
+                task.headers,
+                self.task_tracking_ttl,
+            )
 
         pipe = self.backend.make_pipeline()
         for bind in binds:
@@ -276,9 +299,10 @@ class SystemService(KartonServiceBase):
             operation_bodies.append(operation_body)
 
         self.backend.register_tasks(tasks)
-        self.backend.produce_logs(
-            operation_bodies, logger_name=KARTON_OPERATIONS_QUEUE, level="INFO"
-        )
+        if self.enable_publish_log:
+            self.backend.produce_logs(
+                operation_bodies, logger_name=KARTON_OPERATIONS_QUEUE, level="INFO"
+            )
 
     def process_routing(self) -> None:
         # Order does matter! task dispatching must be before

{karton_core-5.6.0.dist-info → karton_core-5.7.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: karton-core
-Version: 5.6.0
+Version: 5.7.0
 Summary: Distributed malware analysis orchestration framework
 Home-page: https://github.com/CERT-Polska/karton
 Classifier: Programming Language :: Python :: 3

karton_core-5.7.0.dist-info/RECORD

@@ -0,0 +1,27 @@
+karton_core-5.7.0-nspkg.pth,sha256=vHa-jm6pBTeInFrmnsHMg9AOeD88czzQy-6QCFbpRcM,539
+karton/core/__init__.py,sha256=QuT0BWZyp799eY90tK3H1OD2hwuusqMJq8vQwpB3kG4,337
+karton/core/__version__.py,sha256=QmHMXVnw5DVPfWzvN7FS1tOhDAesdxpM_aVOh9CMuSk,22
+karton/core/backend.py,sha256=_IOjN9pWdSBsDnTMYvg-Fpm6Ag-uf2Jb9LWmrtZqVAU,38773
+karton/core/base.py,sha256=lqVJvCHRMzvIOpS8SaWlOaSSJBEVkNQe0oClZC_GQYM,8225
+karton/core/config.py,sha256=M3dB0XgnUO5VzUcGyQa7FyKzmdgmDml1MrzG6CxEuvE,8100
+karton/core/exceptions.py,sha256=8i9WVzi4PinNlX10Cb-lQQC35Hl-JB5R_UKXa9AUKoQ,153
+karton/core/inspect.py,sha256=aIJQEOEkD5q2xLlV8nhxY5qL5zqcnprP-2DdP6ecKlE,6150
+karton/core/karton.py,sha256=l3joJWw8m23wlOErkcQmNFYhLFA5x2la6L0WopxJ7mk,15435
+karton/core/logger.py,sha256=J3XAyG88U0cwYC9zR6E3QD1uJenrQh7zS9-HgxhqeAs,2040
+karton/core/main.py,sha256=ir1-dhn3vbwfh2YHiM6ZYfRBbjwLvJSz0d8tuK1mb_4,8310
+karton/core/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+karton/core/query.py,sha256=sf24DweVlXfJuBbBD_ns2LXhOV-IBwuPG3jBfTJu77s,12063
+karton/core/resource.py,sha256=9kWXpMBRfudH0_whJfSSI27K3Gwv2u93CVa7p68Q5UM,20842
+karton/core/task.py,sha256=gW1szMi5PN2Y06X-Ryo7cmEVluZv1r7W5tvmwIJiD94,18808
+karton/core/test.py,sha256=cj6W4gNt0BpRjsYiiBt0hPE8dmRfUeIc8sSVkxB50cU,9123
+karton/core/utils.py,sha256=sEVqGdVPyYswWuVn8wYXBQmln8Az826N_2HgC__pmW8,4090
+karton/system/__init__.py,sha256=JF51OqRU_Y4c0unOulvmv1KzSHSq4ZpXU8ZsH4nefRM,63
+karton/system/__main__.py,sha256=QJkwIlSwaPRdzwKlNmCAL41HtDAa73db9MZKWmOfxGM,56
+karton/system/system.py,sha256=d_5hhLTthJdr_4gZEGQ6Y-kHvxeBqyQxjjx_wRs3xMA,17285
+karton_core-5.7.0.dist-info/LICENSE,sha256=o8h7hYhn7BJC_-DmrfqWwLjaR_Gbe0TZOOQJuN2ca3I,1519
+karton_core-5.7.0.dist-info/METADATA,sha256=MrmtycTaYsNB8v0LRyuLIHL2bV17n1Lt6e-ak4RfrH8,6818
+karton_core-5.7.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+karton_core-5.7.0.dist-info/entry_points.txt,sha256=OgLlsXy61GP6-Yob3oXqeJ2hlRU6LBLj33fr0NufKz0,98
+karton_core-5.7.0.dist-info/namespace_packages.txt,sha256=X8SslCPsqXDCnGZqrYYolzT3xPzJMq1r-ZQSc0jfAEA,7
+karton_core-5.7.0.dist-info/top_level.txt,sha256=X8SslCPsqXDCnGZqrYYolzT3xPzJMq1r-ZQSc0jfAEA,7
+karton_core-5.7.0.dist-info/RECORD,,

karton_core-5.6.0.dist-info/RECORD

@@ -1,27 +0,0 @@
-karton_core-5.6.0-nspkg.pth,sha256=vHa-jm6pBTeInFrmnsHMg9AOeD88czzQy-6QCFbpRcM,539
-karton/core/__init__.py,sha256=QuT0BWZyp799eY90tK3H1OD2hwuusqMJq8vQwpB3kG4,337
-karton/core/__version__.py,sha256=9eGec3AYz2CNznnfEwKYRiJ65G9gDWtXmb587ljDycg,22
-karton/core/backend.py,sha256=HlAolngYUTx2ajXhLf5RbWxmPOq5orLfz52iTkNxxqM,38147
-karton/core/base.py,sha256=C6Lco3E0XCsxvEjeVOLR9fxh_IWJ1vjC9BqUYsQyewE,8083
-karton/core/config.py,sha256=7oKchitq6pWzPuXRfjBXqVT_BgGIz2p-CDo1RGaNJQg,8118
-karton/core/exceptions.py,sha256=8i9WVzi4PinNlX10Cb-lQQC35Hl-JB5R_UKXa9AUKoQ,153
-karton/core/inspect.py,sha256=aIJQEOEkD5q2xLlV8nhxY5qL5zqcnprP-2DdP6ecKlE,6150
-karton/core/karton.py,sha256=Fi3wNqMGiKvHN2BECsqsvfxkiyuwPdlC21jpqQdkeak,15434
-karton/core/logger.py,sha256=J3XAyG88U0cwYC9zR6E3QD1uJenrQh7zS9-HgxhqeAs,2040
-karton/core/main.py,sha256=ir1-dhn3vbwfh2YHiM6ZYfRBbjwLvJSz0d8tuK1mb_4,8310
-karton/core/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-karton/core/query.py,sha256=sf24DweVlXfJuBbBD_ns2LXhOV-IBwuPG3jBfTJu77s,12063
-karton/core/resource.py,sha256=tA3y_38H9HVKIrCeAU70zHUkQUv0BuCQWMC470JLxxc,20321
-karton/core/task.py,sha256=gW1szMi5PN2Y06X-Ryo7cmEVluZv1r7W5tvmwIJiD94,18808
-karton/core/test.py,sha256=tms-YM7sUKQDHN0vm2_W7DIvHnO_ld_VPsWHnsbKSfk,9102
-karton/core/utils.py,sha256=sEVqGdVPyYswWuVn8wYXBQmln8Az826N_2HgC__pmW8,4090
-karton/system/__init__.py,sha256=JF51OqRU_Y4c0unOulvmv1KzSHSq4ZpXU8ZsH4nefRM,63
-karton/system/__main__.py,sha256=QJkwIlSwaPRdzwKlNmCAL41HtDAa73db9MZKWmOfxGM,56
-karton/system/system.py,sha256=v2rEJYN2Vq5-hH8yUGFv6r2ZdX-TOQg0ldsEC2T6qO8,16315
-karton_core-5.6.0.dist-info/LICENSE,sha256=o8h7hYhn7BJC_-DmrfqWwLjaR_Gbe0TZOOQJuN2ca3I,1519
-karton_core-5.6.0.dist-info/METADATA,sha256=Atu4wfGAqbkw-lVDV_Wtt3fxSG7nV0QmVqjGnfNAFZ4,6818
-karton_core-5.6.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-karton_core-5.6.0.dist-info/entry_points.txt,sha256=OgLlsXy61GP6-Yob3oXqeJ2hlRU6LBLj33fr0NufKz0,98
-karton_core-5.6.0.dist-info/namespace_packages.txt,sha256=X8SslCPsqXDCnGZqrYYolzT3xPzJMq1r-ZQSc0jfAEA,7
-karton_core-5.6.0.dist-info/top_level.txt,sha256=X8SslCPsqXDCnGZqrYYolzT3xPzJMq1r-ZQSc0jfAEA,7
-karton_core-5.6.0.dist-info/RECORD,,