assemblyline-v4-service 4.6.0.dev11__py3-none-any.whl → 4.6.0.2__py3-none-any.whl

assemblyline_v4_service/VERSION

@@ -1 +1 @@
-4.6.0.dev11
+4.6.0.2

assemblyline_v4_service/common/utils.py

@@ -80,14 +80,14 @@ def __extract_passwords_from_lines(texts, password_word, password_regex):
     password_keyword = f"{password_word}:"
     for line in texts:
         if password_keyword in line.lower():
-            new_passwords = re.split(password_regex, line)
+            new_passwords = set(re.split(password_regex, line))
             index = line.lower().rindex(password_keyword)
             if index > 0 and line[index - 1] != " ":
                 special_char = line[index - 1]
                 if special_char in BRACKET_PAIRS:
                     special_char = BRACKET_PAIRS[special_char]
-                for password in new_passwords:
-                    new_passwords.extend([password[:i] for i, ltr in enumerate(password) if ltr == special_char])
+                for password in list(new_passwords):
+                    new_passwords.update([password[:i] for i, ltr in enumerate(password) if ltr == special_char])
 
                 new_passwords = set(new_passwords)
                 new_passwords.discard("")
@@ -121,7 +121,8 @@ def _is_dev_mode() -> bool:
         stack_trace.seek(0)
         read_stack_trace = stack_trace.read()
 
-        if any(msg in read_stack_trace for msg in ['run_service_once', 'pytest', 'assemblyline_v4_service.testing.helper']):
+        if any(msg in read_stack_trace
+               for msg in ['run_service_once', 'pytest', 'assemblyline_v4_service.testing.helper']):
             return True
 
     return False

assemblyline_v4_service/dev/run_service_once.py

@@ -1,6 +1,5 @@
 import argparse
 import cProfile
-import importlib
 import json
 import logging
 import os
@@ -9,8 +8,6 @@ import shutil
 import tempfile
 from typing import Dict, Union
 
-from cart import get_metadata_only, unpack_stream
-
 from assemblyline.common import forge
 from assemblyline.common.heuristics import HeuristicHandler, InvalidHeuristicException
 from assemblyline.common.importing import load_module_by_path
@@ -22,11 +19,12 @@ from assemblyline.odm.models.service import Service
 from assemblyline_v4_service.common.base import ServiceBase
 from assemblyline_v4_service.common.helper import get_heuristics, get_service_manifest
 from assemblyline_v4_service.dev.updater import load_rules
+from cart import get_metadata_only, unpack_stream
 
 
 class RunService:
     def __init__(self):
-        self.service: ServiceBase = None
+        self.service: Union[ServiceBase, None] = None
         self.service_class = None
         self.submission_params = None
         self.file_dir = None
@@ -185,7 +183,7 @@ class RunService:
         LOG.info(f"Cleaning up file used for temporary processing: {target_file}")
         os.unlink(target_file)
 
-        if self.service.rules_directory:
+        if self.service.rules_directory and self.service.rules_directory != "/":
             LOG.info("Cleaning up downloaded signatures..")
             shutil.rmtree(self.service.rules_directory)
 

assemblyline_v4_service/run_privileged_service.py

@@ -3,23 +3,22 @@ import json
 import os
 import shutil
 import tempfile
-import yaml
-
-from json import JSONDecodeError
 from io import BytesIO
+from json import JSONDecodeError
+
+import yaml
+from assemblyline_core.server_base import ServerBase
+from assemblyline_core.tasking_client import TaskingClient
 
-from assemblyline.common import forge
 from assemblyline.common.digests import get_sha256_for_file
 from assemblyline.common.importing import load_module_by_path
 from assemblyline.common.metrics import MetricsFactory
 from assemblyline.common.str_utils import StringTable
-from assemblyline.common.version import FRAMEWORK_VERSION, SYSTEM_VERSION, BUILD_MINOR
+from assemblyline.common.version import BUILD_MINOR, FRAMEWORK_VERSION, SYSTEM_VERSION
 from assemblyline.filestore import FileStoreException
-from assemblyline.remote.datatypes import get_client
 from assemblyline.odm.messages.service_heartbeat import Metrics
 from assemblyline.odm.messages.task import Task as ServiceTask
-from assemblyline_core.tasking_client import TaskingClient
-from assemblyline_core.server_base import ServerBase
+from assemblyline.remote.datatypes import get_client
 from assemblyline_v4_service.common.base import is_recoverable_runtime_error
 
 SERVICE_PATH = os.environ['SERVICE_PATH']
@@ -74,7 +73,7 @@ class RunPrivilegedService(ServerBase):
 
         self.status = STATUSES.INITIALIZING
         self.metric_factory = None
-        
+
     def _load_manifest(self):
         bio = BytesIO()
         with open(SERVICE_MANIFEST, "rb") as srv_manifest:
@@ -126,6 +125,7 @@ class RunPrivilegedService(ServerBase):
 
         # Load on-disk manifest for bootstrap/registration
         service_manifest = self._load_manifest()
+        file_required = service_manifest.get('file_required', True)
 
         # Register the service
         registration = self.tasking_client.register_service(service_manifest)
@@ -146,7 +146,6 @@ class RunPrivilegedService(ServerBase):
         self.service_tool_version = self.service.get_tool_version()
         self.metric_factory = MetricsFactory('service', Metrics, name=self.service_name,
                                              export_zero=False, redis=self.redis)
-        file_required = self.service_config.get('file_required', True)
 
         # Start the service
         self.service.start_service()

assemblyline_v4_service/updater/helper.py

@@ -12,6 +12,7 @@ import psutil
 import regex as re
 import requests
 from git import Repo
+from azure.identity import DefaultAzureCredential
 
 from assemblyline.common.digests import get_sha256_for_file
 from assemblyline.common.identify import Identify
@@ -187,6 +188,7 @@ def git_clone_repo(source: Dict[str, Any], previous_update: int = None, logger=N
     name = source['name']
     url = source['uri']
     key = source.get('private_key', None)
+    use_managed_identity = source.get('use_managed_identity', False)
     username = source.get('username', None)
     password = source.get('password', None)
     branch = source.get('git_branch', None) or None
@@ -195,15 +197,29 @@ def git_clone_repo(source: Dict[str, Any], previous_update: int = None, logger=N
     ca_cert = source.get("ca_cert")
     proxy = source.get('proxy', None)
     auth = None
-    if username and password:
+    git_env = {}
+
+    if use_managed_identity:
+        # Get Azure managed identity token
+        try:
+            credential = DefaultAzureCredential()
+        except Exception as e:
+            logger.warning(f"No managed identity available: {str(e)}")
+            raise SkipSource()
+        # Get token for Azure DevOps scope
+        token = credential.get_token("499b84ac-1321-427f-aa17-267ca6975798/.default")
+
+        git_env['GIT_CONFIG_COUNT'] = '1'
+        git_env['GIT_CONFIG_KEY_0'] = 'http.extraheader'
+        git_env['GIT_CONFIG_VALUE_0'] = f'AUTHORIZATION: bearer {token.token}'
+        auth = None
+    elif username and password:
         # Basic authentication scheme
         auth = f'{username}:{password}@'
     elif password:
         # Token-based authentication
         auth = f'{password}@'
 
-    git_env = {}
-
     if ignore_ssl_errors:
         git_env['GIT_SSL_NO_VERIFY'] = '1'
 

{assemblyline_v4_service-4.6.0.dev11.dist-info → assemblyline_v4_service-4.6.0.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline-v4-service
-Version: 4.6.0.dev11
+Version: 4.6.0.2
 Summary: Assemblyline 4 - Service base
 Home-page: https://github.com/CybercentreCanada/assemblyline-v4-service/
 Author: CCCS Assemblyline development team

{assemblyline_v4_service-4.6.0.dev11.dist-info → assemblyline_v4_service-4.6.0.2.dist-info}/RECORD

@@ -1,8 +1,8 @@
-assemblyline_v4_service/VERSION,sha256=mW2xnGr6BfiMhQqKkN1F1tSNmENG9kfwvfQQ2l0KRGE,12
+assemblyline_v4_service/VERSION,sha256=oxoj6kFkfXeSNEzzudVBzRw3Nm6upadZpg04xAdLLWY,8
 assemblyline_v4_service/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 assemblyline_v4_service/healthz.py,sha256=3QGBg0EZuXC6UN411HFwpLNEop9UvS9feFhvBUTP-k4,1576
 assemblyline_v4_service/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-assemblyline_v4_service/run_privileged_service.py,sha256=IRqhG5ULKjOuy7-W4Fm_R0xb3HPhBlF5iHwNyRBHIME,14535
+assemblyline_v4_service/run_privileged_service.py,sha256=un2zcZjQVKYwMWihLLmeUc3IMJ6ALnFbR1FPeMW1U2A,14486
 assemblyline_v4_service/run_service.py,sha256=XfdABk3hEZsIw31tmFcJc-FbcxvBF9tiDIlg9oHCtZA,5900
 assemblyline_v4_service/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 assemblyline_v4_service/common/api.py,sha256=Xzp8j4HCCfjPvNSGKiZl5ttH2_Itg47cjlH0NXNtth0,6849
@@ -13,18 +13,18 @@ assemblyline_v4_service/common/ontology_helper.py,sha256=9Ad81qbddg_pRMupT8o_Kzx
 assemblyline_v4_service/common/request.py,sha256=W7fqC2xQE3i5i2jlCDyUDp3ZqJQQqSshNW0mQfJMkFg,11792
 assemblyline_v4_service/common/result.py,sha256=9AqM6qCYiia_Bpyn_fBFhzNQMcqJbtFSiGjp57fXW2E,32713
 assemblyline_v4_service/common/task.py,sha256=dJsvRpW0x88CCF_LW6w87jQ_UKTVaOs2Gb117IDNiU8,14233
-assemblyline_v4_service/common/utils.py,sha256=k2__d-V5LjB6o2IKbjVe7tJWKcKuUHto5TyT5oKhIa0,3890
+assemblyline_v4_service/common/utils.py,sha256=umEtTF92Q_Boq0gaIaY8twlwNHAmQOK9HWN6piWR42w,3916
 assemblyline_v4_service/dev/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-assemblyline_v4_service/dev/run_service_once.py,sha256=wx_82hGwavxWhMsmNDfmlYs4KbcLnqeMg91ZGfXTZV0,10621
+assemblyline_v4_service/dev/run_service_once.py,sha256=W9kR49IUbkt8tNXjCT40ZMh-8p5W_odxlkDx6nhTAYM,10656
 assemblyline_v4_service/dev/updater.py,sha256=b-FK6XPRZbETbl-SIYEhnYGT-W7EcQhnxwD6x2NMC7g,6411
 assemblyline_v4_service/updater/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 assemblyline_v4_service/updater/__main__.py,sha256=9Os-u8Tf7MD73JSrUSPmOaErTgfvesNLiEeszU4ujXA,133
 assemblyline_v4_service/updater/app.py,sha256=Mtmx4bkXfP4nFqqa5q15jW8QIXr4JK84lCovxAVyvPs,3317
 assemblyline_v4_service/updater/client.py,sha256=tLY84gaGdFBVIDaMgRHIEa7x2S8jBl7lQLzp4seC6aI,11200
 assemblyline_v4_service/updater/gunicorn_config.py,sha256=p3j2KPBeD5jvMw9O5i7vAtlRgPSVVxIG9AO0DfN82J8,1247
-assemblyline_v4_service/updater/helper.py,sha256=Zy6OBmbTh0YurW0MnM0wM92vaKYMbo_MKnafe_5ONUI,10034
+assemblyline_v4_service/updater/helper.py,sha256=GeXrfuPQL0RB5IGcgvx30vcCEehUueorw6SMSExjE9Q,10751
 assemblyline_v4_service/updater/updater.py,sha256=kli-5v1uVmk2FARAI9DsZ9YM4EhgirkmWJaMJWdm9GI,31795
-assemblyline_v4_service-4.6.0.dev11.dist-info/licenses/LICENCE.md,sha256=NSkYo9EH8h5oOkzg4VhjAHF4339MqPP2cQ8msTPgl-c,1396
+assemblyline_v4_service-4.6.0.2.dist-info/licenses/LICENCE.md,sha256=NSkYo9EH8h5oOkzg4VhjAHF4339MqPP2cQ8msTPgl-c,1396
 test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/conftest.py,sha256=W3SieQpZsZpGEmtLqY4aIlxREDSsHceyCrFcFsWUM0U,1851
 test/test_healthz.py,sha256=DkeLUlrb7rGx3nZ04aADU9HXXu5mZTf_DBwT0xhzIv4,7
@@ -40,7 +40,7 @@ test/test_common/test_request.py,sha256=Ceyds8BNO1O0f1kH1VEb84faJcaupvSjVKIrGdHe
 test/test_common/test_result.py,sha256=6BiOKxEPrKBjOY44jv3TY-yiXm0qI1ok_CZBnjP9TM4,45447
 test/test_common/test_task.py,sha256=P44mNcSe-3tJgDk9ppN3KbM7oN4LBVIuhONG-Gveh74,19007
 test/test_common/test_utils.py,sha256=TbnBxqpS_ZC5ptXR9XJX3xtbItD0mTbtiBxxdyP8J5k,5904
-assemblyline_v4_service-4.6.0.dev11.dist-info/METADATA,sha256=0xjORVh5WgGnnvfyJWucTIt_Tfe6s3HOWoV1BXncpK0,5624
-assemblyline_v4_service-4.6.0.dev11.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-assemblyline_v4_service-4.6.0.dev11.dist-info/top_level.txt,sha256=LpTOEaVCatkrvbVq3EZseMSIa2PQZU-2rhuO_FTpZgY,29
-assemblyline_v4_service-4.6.0.dev11.dist-info/RECORD,,
+assemblyline_v4_service-4.6.0.2.dist-info/METADATA,sha256=1Vl8sqdYaXOgUPEq35RTfFYjmm2aS62foaxCdYhbQYc,5620
+assemblyline_v4_service-4.6.0.2.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
+assemblyline_v4_service-4.6.0.2.dist-info/top_level.txt,sha256=LpTOEaVCatkrvbVq3EZseMSIa2PQZU-2rhuO_FTpZgY,29
+assemblyline_v4_service-4.6.0.2.dist-info/RECORD,,

{assemblyline_v4_service-4.6.0.dev11.dist-info → assemblyline_v4_service-4.6.0.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.0)
+Generator: setuptools (80.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any