assemblyline-v4-service 4.5.1.dev10__py3-none-any.whl → 4.5.1.dev46__py3-none-any.whl

assemblyline_v4_service/VERSION

@@ -1 +1 @@
-4.5.1.dev10
+4.5.1.dev46

assemblyline_v4_service/common/ocr.py

@@ -207,8 +207,17 @@ def detections(ocr_output: str) -> Dict[str, List[str]]:
             # Legacy support (before configurable indicator thresholds)
             terms = indicator_config
         elif isinstance(indicator_config, dict):
-            # Set indicator threshold before variable overwrite with terms list
-            terms = indicator_config.get("terms", [])
+            # Either you're exclusively overwriting the terms list or you're selectively including/excluding terms
+            if indicator_config.get("terms"):
+                # Overwrite terms list with service configuration
+                terms = indicator_config["terms"]
+            else:
+                included_terms = set(indicator_config.get("include", []))
+                excluded_terms = set(indicator_config.get("exclude", []))
+                # Compute the new terms list for indicator type
+                terms = list(set(terms).union(included_terms) - excluded_terms)
+
+            # Set the indicator hit threshold
             hit_threshold = indicator_config.get("threshold", 1)
 
         # Perform a pre-check to see if the terms even exist in the OCR text

assemblyline_v4_service/updater/client.py

@@ -1,16 +1,124 @@
 import os
 
+from assemblyline.datastore.collection import ESCollection
+from assemblyline.odm.models.badlist import Badlist as BadlistModel
+from assemblyline.odm.models.safelist import Safelist as SafelistModel
 from assemblyline.odm.models.signature import Signature as SignatureModel
 from assemblyline_core.badlist_client import BadlistClient
 from assemblyline_core.safelist_client import SafelistClient
 from assemblyline_core.signature_client import SignatureClient
 
-from typing import Any, Dict, List, Union
+from typing import Any, Dict, List, Optional, Set, Union
 
 SIGNATURE_UPDATE_BATCH = int(os.environ.get('SIGNATURE_UPDATE_BATCH', '1000'))
 
 
-class SyncableSignature(SignatureClient):
+class SyncableBadlistClient(BadlistClient):
+    def __init__(self, datastore, config=None):
+        super().__init__(datastore, config)
+        self.sync = False
+
+    def add_update_many(self, data: List[Union[dict, BadlistModel]]) -> Dict[str, Any]:
+        return hashlist_add_update_many(self, self.datastore.badlist, data)
+
+
+class SyncableSafelistClient(SafelistClient):
+    def __init__(self, datastore, config=None):
+        super().__init__(datastore, config)
+        self.sync = False
+
+    def add_update_many(self, data: List[Union[dict, SafelistModel]]) -> Dict[str, Any]:
+        return hashlist_add_update_many(self, self.datastore.safelist, data)
+
+
+def hashlist_add_update_many(client: Union[SyncableBadlistClient, SyncableSafelistClient],
+                             collection: ESCollection,
+                             data: List[Union[dict, BadlistModel, SafelistModel]]):
+    # This generic function allows to sync hashlist items with the system by making direct changes to the datastore
+    # Items that no longer exist at the source will be removed from `sources` to maintain active synchronicity
+    # amongst multiple sources.
+    # If there are no more sources actively blocking the item, then the item will be DISABLED but users can always
+    # re-deploy item if desired
+
+    if not data:
+        return {"success": 0, "errors": False}
+
+    current_ids: Set[str] = set()
+    source: Optional[str] = None
+
+    # Iterate over the list of signatures given
+    for i, d in enumerate(data):
+        if isinstance(d, collection.model_class):
+            d = d.as_primitives()
+
+        if not source:
+            # Set the source name
+            source = d["sources"][0]["name"]
+
+        if client.sync:
+            # Compute the expected ID and add it to the list
+            current_ids.add(client._preprocess_object(d))
+        # Update with JSON-friendly version of data to be sent to API
+        data[i] = d
+
+    if client.sync:
+        # Get the list of items that currently exists in the system for the source
+        existing_ids = set(
+            [
+                i["id"]
+                for i in collection.stream_search(
+                    f"sources.name:{source}", fl="id", as_obj=False
+                )
+            ]
+        )
+
+        # Find the IDs that don't exist at this source anymore and remove the source from the source list
+        missing_ids = existing_ids - current_ids
+        for missing_id in missing_ids:
+            missing_item = collection.get(missing_id)
+            original_sources = missing_item.sources
+            missing_item.sources = [
+                s
+                for s in missing_item.sources
+                if not (s.type == "external" and s.name == source)
+            ]
+
+            # If there are no more sources to back this item, then disable it but leave at least one source
+            if not missing_item.sources:
+                missing_item.enabled = False
+                missing_item.sources = original_sources[:1]
+
+            # Update the last updated time
+            missing_item.updated = "NOW"
+
+            # Update missing item with latest changes
+            collection.save(missing_id, missing_item)
+
+    # Proceed with adding/updating items
+    if len(data) < SIGNATURE_UPDATE_BATCH:
+        # Update all of them in a single batch
+        return super(client.__class__, client).add_update_many(data)
+    else:
+        response = {"success": 0, "errors": False}
+
+        def update_response(r: Dict[str, Any]):
+            # Response has to be in the same format, but show the accumulation of batches
+            response["success"]: int = response["success"] + r["success"]
+            response["errors"]: bool = response["errors"] or r["errors"]
+
+        # Split up data into batches to avoid server timeouts handling requests
+        batch_num = 0
+        start = batch_num * SIGNATURE_UPDATE_BATCH
+        while start < len(data):
+            end = (batch_num + 1) * SIGNATURE_UPDATE_BATCH
+            update_response(super(client.__class__, client).add_update_many(data[start:end]))
+            batch_num += 1
+            start = batch_num * SIGNATURE_UPDATE_BATCH
+
+        return response
+
+
+class SyncableSignatureClient(SignatureClient):
     def __init__(self, datastore, config=None):
         super().__init__(datastore, config)
         self.sync = False
@@ -93,6 +201,19 @@ class SyncableSignature(SignatureClient):
 class UpdaterClient(object):
     def __init__(self, datastore) -> None:
         self.datastore = datastore
-        self.badlist = BadlistClient(datastore)
-        self.safelist = SafelistClient(datastore)
-        self.signature = SyncableSignature(datastore)
+        self._sync = False
+        self.badlist = SyncableBadlistClient(datastore)
+        self.safelist = SyncableSafelistClient(datastore)
+        self.signature = SyncableSignatureClient(datastore)
+
+    @property
+    def sync(self):
+        return self._sync
+
+    @sync.setter
+    def sync(self, value: bool):
+        # Set sync state across clients
+        self.badlist.sync = value
+        self.safelist.sync = value
+        self.signature.sync = value
+        self._sync = value

assemblyline_v4_service/updater/helper.py

@@ -35,6 +35,9 @@ class SkipSource(RuntimeError):
 def add_cacert(cert: str) -> None:
     # Add certificate to requests
     cafile = certifi.where()
+    with open(cafile, 'r') as ca_reader:
+        if cert in ca_reader.read():
+            return
     with open(cafile, 'a') as ca_editor:
         ca_editor.write(f"\n{cert}")
 

assemblyline_v4_service/updater/updater.py

@@ -371,8 +371,8 @@ class ServiceUpdater(ThreadedCoreBase):
                     source = source_obj.as_primitives()
                     uri: str = source['uri']
                     default_classification = source.get('default_classification', classification.UNRESTRICTED)
-                    # Enable signature syncing if the source specifies it
-                    self.client.signature.sync = source.get('sync', False)
+                    # Enable syncing if the source specifies it
+                    self.client.sync = source.get('sync', False)
 
                     try:
                         self.push_status("UPDATING", "Pulling..")

{assemblyline_v4_service-4.5.1.dev10.dist-info → assemblyline_v4_service-4.5.1.dev46.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-v4-service
-Version: 4.5.1.dev10
+Version: 4.5.1.dev46
 Summary: Assemblyline 4 - Service base
 Home-page: https://github.com/CybercentreCanada/assemblyline-v4-service/
 Author: CCCS Assemblyline development team
@@ -24,7 +24,7 @@ Requires-Dist: assemblyline-core
 Requires-Dist: cart
 Requires-Dist: fuzzywuzzy
 Requires-Dist: pefile
-Requires-Dist: pillow ==10.2.0
+Requires-Dist: pillow !=10.1.0,!=10.2.0
 Requires-Dist: python-Levenshtein
 Requires-Dist: regex
 Provides-Extra: updater
@@ -242,9 +242,7 @@ To test an Assemblyline service in standalone mode, the [run_service_once.py](ht
 2. From a terminal, run the `run_service_once` script, where `<service path>` is the path to the service module and `<file path>` is the path of the file to be processed
 
    ```shell
-   python3.9 -m assemblyline_v4_service.dev.run_service_once <service path> <file path>
-   ```
-   python3.7 -m assemblyline_v4_service.dev.run_service_once <service path> <file path>
+   python3.11 -m assemblyline_v4_service.dev.run_service_once <service path> <file path>
    ```
 
 
@@ -261,10 +259,7 @@ To test an Assemblyline service in standalone mode, the [run_service_once.py](ht
 2. From a terminal, run the `run_service_once` script
 
    ```shell
-   python3.9 -m assemblyline_v4_service.dev.run_service_once assemblyline_result_sample_service.result_sample.ResultSample /home/ubuntu/testfile.doc
-   ```
-   python3.7 -m assemblyline_v4_service.dev.run_service_once assemblyline_result_sample_service.result_sample.ResultSample /home/ubuntu/testfile.doc
+   python3.11 -m assemblyline_v4_service.dev.run_service_once assemblyline_result_sample_service.result_sample.ResultSample /home/ubuntu/testfile.doc
    ```
 
-
 3. The `results.json` and any extracted/supplementary files will be outputted to `/home/ubuntu/testfile_resultsample`

{assemblyline_v4_service-4.5.1.dev10.dist-info → assemblyline_v4_service-4.5.1.dev46.dist-info}/RECORD

@@ -1,4 +1,4 @@
-assemblyline_v4_service/VERSION,sha256=MJm_8v8CsbL_j67ZEe8Zv_rM-VewUZr4CgC4PU0CoUg,12
+assemblyline_v4_service/VERSION,sha256=sQM09FS0BK9FrsHaapzAtmW4sNxx-dHBnjL5R7rkljk,12
 assemblyline_v4_service/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 assemblyline_v4_service/healthz.py,sha256=sS1cFkDLw8hUPMpj7tbHXFv8ZmHcazrwZ0l6oQDwwkQ,1575
 assemblyline_v4_service/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -8,7 +8,7 @@ assemblyline_v4_service/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5
 assemblyline_v4_service/common/api.py,sha256=Xzp8j4HCCfjPvNSGKiZl5ttH2_Itg47cjlH0NXNtth0,6849
 assemblyline_v4_service/common/base.py,sha256=mKkkzbxVL_wVMy_VieU9mlHYLqZXndga_4dWWbnEnx8,14045
 assemblyline_v4_service/common/helper.py,sha256=xs9quuf-M1JOdKieBqOmWaOece0CtzXFhhe85xQYmuY,3289
-assemblyline_v4_service/common/ocr.py,sha256=XHHD0bOKu7DdBEMQ3sPZxQjp4K4G3js_Ib2GKWD6Xuw,7897
+assemblyline_v4_service/common/ocr.py,sha256=A8OnjpEor-S3OUC_jZzJ-Er3KKAsMdQLEXTtMS81Xbk,8397
 assemblyline_v4_service/common/ontology_helper.py,sha256=QpwerYoS5hXjWzpx3Pmwv6j2330PQVYqxYGamjcpW3I,7890
 assemblyline_v4_service/common/request.py,sha256=XXBafAQCV43_OBLXOSHxYoDHmqwERBkNul8fb_X6Ves,11774
 assemblyline_v4_service/common/result.py,sha256=9AqM6qCYiia_Bpyn_fBFhzNQMcqJbtFSiGjp57fXW2E,32713
@@ -19,11 +19,12 @@ assemblyline_v4_service/dev/run_service_once.py,sha256=4K3ljw0MnfPGw0-6lzc_vtUYg
 assemblyline_v4_service/updater/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 assemblyline_v4_service/updater/__main__.py,sha256=9Os-u8Tf7MD73JSrUSPmOaErTgfvesNLiEeszU4ujXA,133
 assemblyline_v4_service/updater/app.py,sha256=Mtmx4bkXfP4nFqqa5q15jW8QIXr4JK84lCovxAVyvPs,3317
-assemblyline_v4_service/updater/client.py,sha256=HNmBKJI9VXsRso8IG3JojWsk9ujXJUu_cSJmKh7oG_k,4772
+assemblyline_v4_service/updater/client.py,sha256=oOOIzh-Q-b5uOTo3C4rVSfQxqUO5Y_ogDmUhvWfm5uo,9518
 assemblyline_v4_service/updater/gunicorn_config.py,sha256=p3j2KPBeD5jvMw9O5i7vAtlRgPSVVxIG9AO0DfN82J8,1247
-assemblyline_v4_service/updater/helper.py,sha256=HbH5p6UTdHyIgoctF1c1pQkoqTtzaxfHOi9KXGwn0eM,9435
-assemblyline_v4_service/updater/updater.py,sha256=yYDME2ir38Qr-Qe8S0uqcYNxjPRsgUrBByhLwg_rweU,29033
+assemblyline_v4_service/updater/helper.py,sha256=-B35wdjpeY4t1R9SPDrTFHFKHwE3uzy9N69mV6mHy-g,9532
+assemblyline_v4_service/updater/updater.py,sha256=5cJH82VTP3VrwoNE97ubKP2dr90AI_T-AKRistvN5rI,29013
 test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+test/conftest.py,sha256=W3SieQpZsZpGEmtLqY4aIlxREDSsHceyCrFcFsWUM0U,1851
 test/test_healthz.py,sha256=DkeLUlrb7rGx3nZ04aADU9HXXu5mZTf_DBwT0xhzIv4,7
 test/test_run_privileged_service.py,sha256=DkeLUlrb7rGx3nZ04aADU9HXXu5mZTf_DBwT0xhzIv4,7
 test/test_run_service.py,sha256=DkeLUlrb7rGx3nZ04aADU9HXXu5mZTf_DBwT0xhzIv4,7
@@ -37,8 +38,8 @@ test/test_common/test_request.py,sha256=wxSwnOj-_YOv2SuZjOJsw09q8A7p8GJmJuK4vozq
 test/test_common/test_result.py,sha256=Wm0Cs5kZRzlZr0jL-l8OTsYAvkoN2eaB3NkeXzvyssI,42208
 test/test_common/test_task.py,sha256=jnfF68EgJIu30Pz_4jiJHkncfI-3XpGaut5r79KIXOA,18718
 test/test_common/test_utils.py,sha256=TbnBxqpS_ZC5ptXR9XJX3xtbItD0mTbtiBxxdyP8J5k,5904
-assemblyline_v4_service-4.5.1.dev10.dist-info/LICENCE.md,sha256=NSkYo9EH8h5oOkzg4VhjAHF4339MqPP2cQ8msTPgl-c,1396
-assemblyline_v4_service-4.5.1.dev10.dist-info/METADATA,sha256=deXIyf91VVK9BRVU09vvD73BwcrirOx6_wXDfLzqIvg,9739
-assemblyline_v4_service-4.5.1.dev10.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-assemblyline_v4_service-4.5.1.dev10.dist-info/top_level.txt,sha256=LpTOEaVCatkrvbVq3EZseMSIa2PQZU-2rhuO_FTpZgY,29
-assemblyline_v4_service-4.5.1.dev10.dist-info/RECORD,,
+assemblyline_v4_service-4.5.1.dev46.dist-info/LICENCE.md,sha256=NSkYo9EH8h5oOkzg4VhjAHF4339MqPP2cQ8msTPgl-c,1396
+assemblyline_v4_service-4.5.1.dev46.dist-info/METADATA,sha256=WMqxrxQMu_e8Zj5DbSRC_p1nA8VhyrDScDlSGGOk63Q,9498
+assemblyline_v4_service-4.5.1.dev46.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+assemblyline_v4_service-4.5.1.dev46.dist-info/top_level.txt,sha256=LpTOEaVCatkrvbVq3EZseMSIa2PQZU-2rhuO_FTpZgY,29
+assemblyline_v4_service-4.5.1.dev46.dist-info/RECORD,,

{assemblyline_v4_service-4.5.1.dev10.dist-info → assemblyline_v4_service-4.5.1.dev46.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.42.0)
+Generator: bdist_wheel (0.43.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

test/conftest.py

@@ -0,0 +1,61 @@
+"""
+Pytest configuration file, setup global pytest fixtures and functions here.
+"""
+import os
+import pytest
+
+from assemblyline.common import forge
+from assemblyline.datastore.helper import AssemblylineDatastore
+from assemblyline.datastore.store import ESStore
+
+original_skip = pytest.skip
+
+# Check if we are in an unattended build environment where skips won't be noticed
+IN_CI_ENVIRONMENT = any(indicator in os.environ for indicator in
+                        ['CI', 'BITBUCKET_BUILD_NUMBER', 'AGENT_JOBSTATUS'])
+
+
+def skip_or_fail(message):
+    """Skip or fail the current test, based on the environment"""
+    if IN_CI_ENVIRONMENT:
+        pytest.fail(message)
+    else:
+        original_skip(message)
+
+
+# Replace the built in skip function with our own
+pytest.skip = skip_or_fail
+
+
+@pytest.fixture(scope='session')
+def config():
+    config = forge.get_config()
+    config.logging.log_level = 'INFO'
+    config.logging.log_as_json = False
+    config.core.metrics.apm_server.server_url = None
+    config.core.metrics.export_interval = 1
+    config.datastore.archive.enabled = True
+    return config
+
+
+@pytest.fixture(scope='module')
+def datastore_connection(config):
+    store = ESStore(config.datastore.hosts)
+    ret_val = store.ping()
+    if not ret_val:
+        pytest.skip("Could not connect to datastore")
+    return AssemblylineDatastore(store)
+
+
+@pytest.fixture(scope='module')
+def clean_datastore(datastore_connection: AssemblylineDatastore):
+    for name in datastore_connection.ds.get_models():
+        datastore_connection.get_collection(name).wipe()
+    return datastore_connection
+
+
+@pytest.fixture(scope='function')
+def function_clean_datastore(datastore_connection: AssemblylineDatastore):
+    for name in datastore_connection.ds.get_models():
+        datastore_connection.get_collection(name).wipe()
+    return datastore_connection