assemblyline-v4-service 4.4.1.dev351__py3-none-any.whl → 4.4.1.dev355__py3-none-any.whl

assemblyline_v4_service/VERSION

@@ -1 +1 @@
-4.4.1.dev351
+4.4.1.dev355

assemblyline_v4_service/common/ocr.py

@@ -1,13 +1,12 @@
 from __future__ import annotations
 
-from typing import TextIO
+from typing import Dict, List, TextIO
 
-import regex
 from assemblyline_v4_service.common.helper import get_service_manifest
 from assemblyline_v4_service.common.utils import PASSWORD_WORDS
 
 # TODO: Would prefer this mapping to be dynamic from trusted sources (ie. import from library), but will copy-paste for now
-OCR_INDICATORS_MAPPING: dict[str, list[str]] = {
+OCR_INDICATORS_TERMS: dict[str, list[str]] = {
     'ransomware': [
         # https://github.com/cuckoosandbox/community/blob/master/modules/signatures/windows/ransomware_message.py
         "your files", "your data", "your documents", "restore files",
@@ -67,16 +66,45 @@ OCR_INDICATORS_MAPPING: dict[str, list[str]] = {
     ]
 }
 
+# The minimum number of indicator hits to avoid FP detections
+OCR_INDICATORS_THRESHOLD: Dict[str, int] = {"ransomware": 2, "macros": 2, "banned": 1, "password": 1}
 
-def ocr_detections(image_path: str, ocr_io: TextIO = None) -> dict[str, list[str]]:
+try:
+    # Retrieve service-configured OCR settings on module load
+    ocr_config: Dict = get_service_manifest().get("config", {}).get("ocr", {})
+    indicators = set(list(OCR_INDICATORS_TERMS.keys()) + list(ocr_config.keys()))
+    for i in indicators:
+        # Backwards compatibility: Check how the OCR configuration is formatted
+        indicator_config = ocr_config.get(i)
+        indicator_terms = []
+        indicator_threshold = 1
+        if not indicator_config:
+            # Empty block/no override provided by service
+            pass
+        elif isinstance(indicator_config, list):
+            # Legacy support (before configurable indicator thresholds)
+            indicator_terms = indicator_config
+            pass
+        elif isinstance(indicator_config, dict):
+            # Set indicator threshold before variable overwrite with terms list
+            indicator_terms = indicator_config.get('terms', [])
+            indicator_threshold = indicator_config.get('threshold', 1)
+        OCR_INDICATORS_TERMS[i] = indicator_terms or OCR_INDICATORS_TERMS.get(i, [])
+        OCR_INDICATORS_THRESHOLD[i] = indicator_threshold or OCR_INDICATORS_THRESHOLD.get(i, 1)
+
+except Exception:
+    pass
+
+
+def ocr_detections(image_path: str, ocr_io: TextIO = None) -> Dict[str, List[str]]:
     try:
         import pytesseract
         from PIL import Image
     except ImportError as exc:
         raise ImportError(
-            'In order to use this method to scan for OCR detections, '
-            'ensure you have the following installed in your service:\n'
-            'tesseract-ocr, pytesseract, and Pillow.\n'
+            "In order to use this method to scan for OCR detections, "
+            "ensure you have the following installed in your service:\n"
+            "tesseract-ocr, pytesseract, and Pillow.\n"
             'You can do this via "apt-get install -y tesseract-ocr" and "pip install Pillow pytesseract"'
         ) from exc
 
@@ -84,7 +112,9 @@ def ocr_detections(image_path: str, ocr_io: TextIO = None) -> dict[str, list[str
     ocr_output = ""
 
     try:
-        ocr_output = pytesseract.image_to_string(Image.open(image_path), timeout=15)  # Stop OCR after 15 seconds
+        ocr_output = pytesseract.image_to_string(
+            Image.open(image_path), timeout=15
+        )  # Stop OCR after 15 seconds
     except (TypeError, RuntimeError):
         # Image given isn't supported therefore no OCR output can be given with tesseract
         return {}
@@ -97,37 +127,51 @@ def ocr_detections(image_path: str, ocr_io: TextIO = None) -> dict[str, list[str
     return detections(ocr_output)
 
 
-def detections(ocr_output: str) -> dict[str, list[str]]:
-    detection_output: dict[str, list[str]] = {}
-    ocr_config: dict[str, list[str]] = {}
+def detections(ocr_output: str) -> Dict[str, List[str]]:
+    indicators = list(OCR_INDICATORS_TERMS.keys())
+    ocr_config = {}
     try:
-        # If running an AL service, grab OCR configuration from service manifest
-        ocr_config = get_service_manifest().get('config', {}).get('ocr', {})
+        # Retrieve service-configured OCR settings on module load
+        ocr_config: Dict = get_service_manifest().get("config", {}).get("ocr", {})
+        indicators = set(list(OCR_INDICATORS_TERMS.keys()) + list(ocr_config.keys()))
     except Exception:
         pass
-    indicators = set(list(OCR_INDICATORS_MAPPING.keys()) + list(ocr_config.keys()))
+
+    detection_output: Dict[str, List[str]] = {}
     # Iterate over the different indicators and include lines of detection in response
     for indicator in indicators:
-        list_of_terms = ocr_config.get(indicator, []) or OCR_INDICATORS_MAPPING.get(indicator, [])
-        if not list_of_terms:
-            # If no terms specified, move onto next indicator
+        # Backwards compatibility: Check how the OCR configuration is formatted
+        indicator_config = ocr_config.get(indicator)
+        terms = OCR_INDICATORS_TERMS.get(indicator, [])
+        hit_threshold = OCR_INDICATORS_THRESHOLD.get(indicator, 1)
+        if not indicator_config:
+            # Empty block/no override provided by service
+            pass
+        elif isinstance(indicator_config, list):
+            # Legacy support (before configurable indicator thresholds)
+            terms = indicator_config
+            pass
+        elif isinstance(indicator_config, dict):
+            # Set indicator threshold before variable overwrite with terms list
+            terms = indicator_config.get('terms', [])
+            hit_threshold = indicator_config.get('threshold', 1)
+
+        # Perform a pre-check to see if the terms even exist in the OCR text
+        if not any([t.lower() in ocr_output.lower() for t in terms]):
             continue
-        indicator_hits: set[str | None] = set()
-        regex_exp = regex.compile(f"({')|('.join(list_of_terms).lower()})")
-        list_of_strings: list[str] = []
-        for line in ocr_output.split('\n'):
-            search = regex_exp.search(line.lower())
-            if search:
-                indicator_hits = indicator_hits.union(set(search.groups()))
-                list_of_strings.append(line)
-        if None in indicator_hits:
-            indicator_hits.remove(None)
-
-        if list_of_strings:
-            if len(indicator_hits) >= 2:
-                # We consider the detection to be credible if there's more than a single indicator hit
-                detection_output[indicator] = list_of_strings
-            if indicator in ['banned', 'password']:
-                # Except if we're dealing with banned/password, one hit is more than enough
-                detection_output[indicator] = list_of_strings
+
+        # Keep a track of the hits and the lines corresponding with term hits
+        indicator_hits: int = 0
+        list_of_strings: List[str] = []
+        for line in ocr_output.split("\n"):
+            for t in terms:
+                term_count = line.lower().count(t.lower())
+                if term_count:
+                    indicator_hits += term_count
+                    if line not in list_of_strings:
+                        list_of_strings.append(line)
+
+        if list_of_strings and indicator_hits >= hit_threshold:
+            # If we were to find hits and those hits are above the required threshold, then add them to output
+            detection_output[indicator] = list_of_strings
     return detection_output

{assemblyline_v4_service-4.4.1.dev351.dist-info → assemblyline_v4_service-4.4.1.dev355.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-v4-service
-Version: 4.4.1.dev351
+Version: 4.4.1.dev355
 Summary: Assemblyline 4 - Service base
 Home-page: https://github.com/CybercentreCanada/assemblyline-v4-service/
 Author: CCCS Assemblyline development team

{assemblyline_v4_service-4.4.1.dev351.dist-info → assemblyline_v4_service-4.4.1.dev355.dist-info}/RECORD

@@ -1,4 +1,4 @@
-assemblyline_v4_service/VERSION,sha256=S4YmcS3JWMqm8BjF0G1J4UYUF8kF9ACn4azroxWQmcc,13
+assemblyline_v4_service/VERSION,sha256=DB7TR909uaAA41K89VQwr4iyPmWMyngEPPHlQ8Nk-E8,13
 assemblyline_v4_service/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 assemblyline_v4_service/healthz.py,sha256=sS1cFkDLw8hUPMpj7tbHXFv8ZmHcazrwZ0l6oQDwwkQ,1575
 assemblyline_v4_service/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -8,7 +8,7 @@ assemblyline_v4_service/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5
 assemblyline_v4_service/common/api.py,sha256=8s2GJI_GmY4ClnjPOn7ijYs6m50cVJfjVx1h3GBQVms,6694
 assemblyline_v4_service/common/base.py,sha256=9xufnspN99J1EHTru1fdkflRwB6PGdfyCUDvYwUIBEk,13610
 assemblyline_v4_service/common/helper.py,sha256=xs9quuf-M1JOdKieBqOmWaOece0CtzXFhhe85xQYmuY,3289
-assemblyline_v4_service/common/ocr.py,sha256=ML9AwlGCeogFGoeLuFdhcu75E5t6fTE69U82fENGRkY,6528
+assemblyline_v4_service/common/ocr.py,sha256=NRK5FqTQZQdjGBj9kSOFyP1Z8gA3lNgk_MK7yeNnJsE,8466
 assemblyline_v4_service/common/ontology_helper.py,sha256=QpwerYoS5hXjWzpx3Pmwv6j2330PQVYqxYGamjcpW3I,7890
 assemblyline_v4_service/common/request.py,sha256=XXBafAQCV43_OBLXOSHxYoDHmqwERBkNul8fb_X6Ves,11774
 assemblyline_v4_service/common/result.py,sha256=9AqM6qCYiia_Bpyn_fBFhzNQMcqJbtFSiGjp57fXW2E,32713
@@ -37,8 +37,8 @@ test/test_common/test_request.py,sha256=wxSwnOj-_YOv2SuZjOJsw09q8A7p8GJmJuK4vozq
 test/test_common/test_result.py,sha256=Wm0Cs5kZRzlZr0jL-l8OTsYAvkoN2eaB3NkeXzvyssI,42208
 test/test_common/test_task.py,sha256=jnfF68EgJIu30Pz_4jiJHkncfI-3XpGaut5r79KIXOA,18718
 test/test_common/test_utils.py,sha256=TbnBxqpS_ZC5ptXR9XJX3xtbItD0mTbtiBxxdyP8J5k,5904
-assemblyline_v4_service-4.4.1.dev351.dist-info/LICENCE.md,sha256=NSkYo9EH8h5oOkzg4VhjAHF4339MqPP2cQ8msTPgl-c,1396
-assemblyline_v4_service-4.4.1.dev351.dist-info/METADATA,sha256=u1l3L4q6HCN1mKbiy2vXVuRwxHQq0HL0AilqRtyh-KI,9663
-assemblyline_v4_service-4.4.1.dev351.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-assemblyline_v4_service-4.4.1.dev351.dist-info/top_level.txt,sha256=LpTOEaVCatkrvbVq3EZseMSIa2PQZU-2rhuO_FTpZgY,29
-assemblyline_v4_service-4.4.1.dev351.dist-info/RECORD,,
+assemblyline_v4_service-4.4.1.dev355.dist-info/LICENCE.md,sha256=NSkYo9EH8h5oOkzg4VhjAHF4339MqPP2cQ8msTPgl-c,1396
+assemblyline_v4_service-4.4.1.dev355.dist-info/METADATA,sha256=56hrcskcjpjwkV4ZvAzqILJAg6FYHzaCG-MzSI91ykk,9663
+assemblyline_v4_service-4.4.1.dev355.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+assemblyline_v4_service-4.4.1.dev355.dist-info/top_level.txt,sha256=LpTOEaVCatkrvbVq3EZseMSIa2PQZU-2rhuO_FTpZgY,29
+assemblyline_v4_service-4.4.1.dev355.dist-info/RECORD,,