assemblyline-v4-service 4.5.1.dev427__py3-none-any.whl → 4.5.1.dev436__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of assemblyline-v4-service might be problematic. Click here for more details.
- assemblyline_v4_service/VERSION +1 -1
- assemblyline_v4_service/common/base.py +1 -2
- assemblyline_v4_service/dev/run_service_once.py +0 -1
- assemblyline_v4_service/healthz.py +1 -1
- assemblyline_v4_service/run_privileged_service.py +2 -4
- assemblyline_v4_service/run_service.py +0 -3
- assemblyline_v4_service/updater/helper.py +5 -2
- assemblyline_v4_service-4.5.1.dev436.dist-info/METADATA +107 -0
- {assemblyline_v4_service-4.5.1.dev427.dist-info → assemblyline_v4_service-4.5.1.dev436.dist-info}/RECORD +12 -12
- assemblyline_v4_service-4.5.1.dev427.dist-info/METADATA +0 -276
- {assemblyline_v4_service-4.5.1.dev427.dist-info → assemblyline_v4_service-4.5.1.dev436.dist-info}/LICENCE.md +0 -0
- {assemblyline_v4_service-4.5.1.dev427.dist-info → assemblyline_v4_service-4.5.1.dev436.dist-info}/WHEEL +0 -0
- {assemblyline_v4_service-4.5.1.dev427.dist-info → assemblyline_v4_service-4.5.1.dev436.dist-info}/top_level.txt +0 -0
assemblyline_v4_service/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
4.5.1.
|
|
1
|
+
4.5.1.dev436
|
|
@@ -26,7 +26,6 @@ from assemblyline_v4_service.common.task import Task
|
|
|
26
26
|
# Ignore all other warnings that a service's libraries can generate
|
|
27
27
|
warnings.filterwarnings("ignore")
|
|
28
28
|
|
|
29
|
-
LOG_LEVEL = logging.getLevelName(os.environ.get("LOG_LEVEL", "INFO"))
|
|
30
29
|
UPDATES_DIR = os.environ.get('UPDATES_DIR', '/updates')
|
|
31
30
|
UPDATES_CA = os.environ.get('UPDATES_CA', '/etc/assemblyline/ssl/al_root-ca.crt')
|
|
32
31
|
PRIVILEGED = os.environ.get('PRIVILEGED', 'false') == 'true'
|
|
@@ -56,7 +55,7 @@ class ServiceBase:
|
|
|
56
55
|
|
|
57
56
|
self.name = self.service_attributes.name.lower()
|
|
58
57
|
# Initialize logging for the service
|
|
59
|
-
log.init_logging(f'{self.service_attributes.name}'
|
|
58
|
+
log.init_logging(f'{self.service_attributes.name}')
|
|
60
59
|
self.log = logging.getLogger(f'assemblyline.service.{self.name}')
|
|
61
60
|
|
|
62
61
|
# Replace warning/error methods with our own patched version
|
|
@@ -20,7 +20,7 @@ from assemblyline.odm.messages.service_heartbeat import Metrics
|
|
|
20
20
|
from assemblyline.odm.messages.task import Task as ServiceTask
|
|
21
21
|
from assemblyline_core.tasking_client import TaskingClient
|
|
22
22
|
from assemblyline_core.server_base import ServerBase
|
|
23
|
-
from assemblyline_v4_service.common.base import
|
|
23
|
+
from assemblyline_v4_service.common.base import is_recoverable_runtime_error
|
|
24
24
|
|
|
25
25
|
SERVICE_PATH = os.environ['SERVICE_PATH']
|
|
26
26
|
SERVICE_TAG = os.environ.get("SERVICE_TAG", f"{FRAMEWORK_VERSION}.{SYSTEM_VERSION}.{BUILD_MINOR}.dev0").encode("utf-8")
|
|
@@ -74,9 +74,7 @@ class RunPrivilegedService(ServerBase):
|
|
|
74
74
|
|
|
75
75
|
self.status = STATUSES.INITIALIZING
|
|
76
76
|
self.metric_factory = None
|
|
77
|
-
|
|
78
|
-
self.log.setLevel(LOG_LEVEL)
|
|
79
|
-
|
|
77
|
+
|
|
80
78
|
def _load_manifest(self):
|
|
81
79
|
bio = BytesIO()
|
|
82
80
|
with open(SERVICE_MANIFEST, "rb") as srv_manifest:
|
|
@@ -8,7 +8,6 @@ from assemblyline.common.version import FRAMEWORK_VERSION, SYSTEM_VERSION, BUILD
|
|
|
8
8
|
from assemblyline.odm.messages.task import Task as ServiceTask
|
|
9
9
|
from assemblyline_core.server_base import ServerBase
|
|
10
10
|
from assemblyline_v4_service.common import helper
|
|
11
|
-
from assemblyline_v4_service.common.base import LOG_LEVEL
|
|
12
11
|
|
|
13
12
|
SERVICE_PATH = os.environ['SERVICE_PATH']
|
|
14
13
|
SERVICE_NAME = SERVICE_PATH.split(".")[-1].lower()
|
|
@@ -39,8 +38,6 @@ class RunService(ServerBase):
|
|
|
39
38
|
self.done_fifo = None
|
|
40
39
|
self.tasking_dir = os.environ.get('TASKING_DIR', tempfile.gettempdir())
|
|
41
40
|
|
|
42
|
-
self.log.setLevel(LOG_LEVEL)
|
|
43
|
-
|
|
44
41
|
def try_run(self):
|
|
45
42
|
try:
|
|
46
43
|
self.service_class = load_module_by_path(SERVICE_PATH)
|
|
@@ -135,8 +135,11 @@ def url_download(source: Dict[str, Any], previous_update: int, logger: Logger, o
|
|
|
135
135
|
else:
|
|
136
136
|
headers = {'If-Modified-Since': previous_update}
|
|
137
137
|
|
|
138
|
-
if fetch_method
|
|
139
|
-
response =
|
|
138
|
+
if fetch_method == 'get':
|
|
139
|
+
response = session.get(uri, auth=auth, headers=headers, proxies=proxies, stream=True)
|
|
140
|
+
elif fetch_method == 'post':
|
|
141
|
+
json = source.get('post_data') or None
|
|
142
|
+
response = session.post(uri, auth=auth, headers=headers, proxies=proxies, json=json, stream=True)
|
|
140
143
|
else:
|
|
141
144
|
raise ValueError(f"Unknown fetch method: {fetch_method}")
|
|
142
145
|
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
|
+
Name: assemblyline-v4-service
|
|
3
|
+
Version: 4.5.1.dev436
|
|
4
|
+
Summary: Assemblyline 4 - Service base
|
|
5
|
+
Home-page: https://github.com/CybercentreCanada/assemblyline-v4-service/
|
|
6
|
+
Author: CCCS Assemblyline development team
|
|
7
|
+
Author-email: assemblyline@cyber.gc.ca
|
|
8
|
+
License: MIT
|
|
9
|
+
Keywords: assemblyline automated malware analysis gc canada cse-cst cse cst cyber cccs
|
|
10
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.7
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENCE.md
|
|
22
|
+
Requires-Dist: assemblyline
|
|
23
|
+
Requires-Dist: assemblyline-core
|
|
24
|
+
Requires-Dist: cart
|
|
25
|
+
Requires-Dist: fuzzywuzzy
|
|
26
|
+
Requires-Dist: pefile
|
|
27
|
+
Requires-Dist: pillow!=10.1.0,!=10.2.0
|
|
28
|
+
Requires-Dist: python-Levenshtein
|
|
29
|
+
Requires-Dist: regex
|
|
30
|
+
Provides-Extra: updater
|
|
31
|
+
Requires-Dist: gunicorn[gevent]; extra == "updater"
|
|
32
|
+
Requires-Dist: flask; extra == "updater"
|
|
33
|
+
Requires-Dist: gitpython; extra == "updater"
|
|
34
|
+
Requires-Dist: git-remote-codecommit; extra == "updater"
|
|
35
|
+
Requires-Dist: psutil; extra == "updater"
|
|
36
|
+
Dynamic: author
|
|
37
|
+
Dynamic: author-email
|
|
38
|
+
Dynamic: classifier
|
|
39
|
+
Dynamic: description
|
|
40
|
+
Dynamic: description-content-type
|
|
41
|
+
Dynamic: home-page
|
|
42
|
+
Dynamic: keywords
|
|
43
|
+
Dynamic: license
|
|
44
|
+
Dynamic: provides-extra
|
|
45
|
+
Dynamic: requires-dist
|
|
46
|
+
Dynamic: summary
|
|
47
|
+
|
|
48
|
+
[](https://discord.gg/GUAy9wErNu)
|
|
49
|
+
[](https://discord.gg/GUAy9wErNu)
|
|
50
|
+
[](https://github.com/CybercentreCanada/assemblyline)
|
|
51
|
+
[](https://github.com/CybercentreCanada/assemblyline-v4-service)
|
|
52
|
+
[](https://github.com/CybercentreCanada/assemblyline/issues?q=is:issue+is:open+label:service-base)
|
|
53
|
+
[](./LICENCE.md)
|
|
54
|
+
|
|
55
|
+
# Assemblyline 4 - Service Base
|
|
56
|
+
|
|
57
|
+
This repository provides the base service functionality for Assemblyline 4 services.
|
|
58
|
+
|
|
59
|
+
## Image variants and tags
|
|
60
|
+
|
|
61
|
+
| **Tag Type** | **Description** | **Example Tag** |
|
|
62
|
+
| :----------: | :----------------------------------------------------------------------------------------------- | :------------------------: |
|
|
63
|
+
| latest | The most recent build (can be unstable). | `latest` |
|
|
64
|
+
| build_type | The type of build used. `dev` is the latest unstable build. `stable` is the latest stable build. | `stable` or `dev` |
|
|
65
|
+
| series | Complete build details, including version and build type: `version.buildType`. | `4.5.stable`, `4.5.1.dev3` |
|
|
66
|
+
|
|
67
|
+
## Creating a new Assemblyline service
|
|
68
|
+
|
|
69
|
+
You can create a new Assemblyline service by using this [template](https://github.com/CybercentreCanada/assemblyline-service-template):
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
apt install jq
|
|
73
|
+
pip install git+https://github.com/CybercentreCanada/assemblyline-service-template.git
|
|
74
|
+
cruft create https://github.com/CybercentreCanada/assemblyline-service-template.git
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
## Documentation
|
|
78
|
+
|
|
79
|
+
For more information about service development for Assemblyline, follow this [guide](https://cybercentrecanada.github.io/assemblyline4_docs/developer_manual/services/developing_an_assemblyline_service/).
|
|
80
|
+
|
|
81
|
+
---
|
|
82
|
+
|
|
83
|
+
# Assemblyline 4 - Service Base
|
|
84
|
+
|
|
85
|
+
Ce référentiel fournit les fonctionnalités de base des services Assemblyline 4.
|
|
86
|
+
|
|
87
|
+
## Créer un nouveau service Assemblyline
|
|
88
|
+
|
|
89
|
+
Vous pouvez créer un nouveau service Assemblyline en utilisant ce [template](https://github.com/CybercentreCanada/assemblyline-service-template).
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
apt install jq
|
|
93
|
+
pip install git+https://github.com/CybercentreCanada/assemblyline-service-template.git
|
|
94
|
+
cruft create https://github.com/CybercentreCanada/assemblyline-service-template.git
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
## Variantes et étiquettes d'image
|
|
98
|
+
|
|
99
|
+
| **Type d'étiquette** | **Description** | **Exemple d'étiquette** |
|
|
100
|
+
| :------------------: | :--------------------------------------------------------------------------------------------------------------- | :------------------------: |
|
|
101
|
+
| dernière | La version la plus récente (peut être instable). | `latest` |
|
|
102
|
+
| build_type | Le type de compilation utilisé. `dev` est la dernière version instable. `stable` est la dernière version stable. | `stable` or `dev` |
|
|
103
|
+
| séries | Le détail de compilation utilisé, incluant la version et le type de compilation : `version.buildType`. | `4.5.stable`, `4.5.1.dev3` |
|
|
104
|
+
|
|
105
|
+
## Documentation
|
|
106
|
+
|
|
107
|
+
Pour plus d'informations sur le développement des services pour Assemblyline, suivez ce [guide](https://cybercentrecanada.github.io/assemblyline4_docs/developer_manual/services/developing_an_assemblyline_service/).
|
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
assemblyline_v4_service/VERSION,sha256=
|
|
1
|
+
assemblyline_v4_service/VERSION,sha256=DFR55ltd2-bWiZqZbNQPaXSNpxRmdD1FRx6hMC2BD6g,13
|
|
2
2
|
assemblyline_v4_service/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
-
assemblyline_v4_service/healthz.py,sha256=
|
|
3
|
+
assemblyline_v4_service/healthz.py,sha256=3QGBg0EZuXC6UN411HFwpLNEop9UvS9feFhvBUTP-k4,1576
|
|
4
4
|
assemblyline_v4_service/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
|
-
assemblyline_v4_service/run_privileged_service.py,sha256=
|
|
6
|
-
assemblyline_v4_service/run_service.py,sha256=
|
|
5
|
+
assemblyline_v4_service/run_privileged_service.py,sha256=IRqhG5ULKjOuy7-W4Fm_R0xb3HPhBlF5iHwNyRBHIME,14535
|
|
6
|
+
assemblyline_v4_service/run_service.py,sha256=XfdABk3hEZsIw31tmFcJc-FbcxvBF9tiDIlg9oHCtZA,5900
|
|
7
7
|
assemblyline_v4_service/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
8
|
assemblyline_v4_service/common/api.py,sha256=Xzp8j4HCCfjPvNSGKiZl5ttH2_Itg47cjlH0NXNtth0,6849
|
|
9
|
-
assemblyline_v4_service/common/base.py,sha256=
|
|
9
|
+
assemblyline_v4_service/common/base.py,sha256=psivTxiOeN2jqL3G3I26oY9JFK-qPuwrg5y_y_d7xYs,14127
|
|
10
10
|
assemblyline_v4_service/common/helper.py,sha256=xs9quuf-M1JOdKieBqOmWaOece0CtzXFhhe85xQYmuY,3289
|
|
11
11
|
assemblyline_v4_service/common/ocr.py,sha256=3fV0PyY3oui_ucAM9dkolP0VRYKACKJuGY4M64DudIE,8841
|
|
12
12
|
assemblyline_v4_service/common/ontology_helper.py,sha256=9Ad81qbddg_pRMupT8o_KzxbKgpodaRqpc3mPoEKLtw,8494
|
|
@@ -15,13 +15,13 @@ assemblyline_v4_service/common/result.py,sha256=9AqM6qCYiia_Bpyn_fBFhzNQMcqJbtFS
|
|
|
15
15
|
assemblyline_v4_service/common/task.py,sha256=dJsvRpW0x88CCF_LW6w87jQ_UKTVaOs2Gb117IDNiU8,14233
|
|
16
16
|
assemblyline_v4_service/common/utils.py,sha256=k2__d-V5LjB6o2IKbjVe7tJWKcKuUHto5TyT5oKhIa0,3890
|
|
17
17
|
assemblyline_v4_service/dev/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
18
|
-
assemblyline_v4_service/dev/run_service_once.py,sha256=
|
|
18
|
+
assemblyline_v4_service/dev/run_service_once.py,sha256=1sdxkRBI7x9Sht8EVCTMIW3MgSxIeV6Ufw77Wx9Vf20,10150
|
|
19
19
|
assemblyline_v4_service/updater/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
20
20
|
assemblyline_v4_service/updater/__main__.py,sha256=9Os-u8Tf7MD73JSrUSPmOaErTgfvesNLiEeszU4ujXA,133
|
|
21
21
|
assemblyline_v4_service/updater/app.py,sha256=Mtmx4bkXfP4nFqqa5q15jW8QIXr4JK84lCovxAVyvPs,3317
|
|
22
22
|
assemblyline_v4_service/updater/client.py,sha256=eBOK8doMu9nsIO6kXWtaBMy4hJIKRtQG0iC_21a5CG4,10849
|
|
23
23
|
assemblyline_v4_service/updater/gunicorn_config.py,sha256=p3j2KPBeD5jvMw9O5i7vAtlRgPSVVxIG9AO0DfN82J8,1247
|
|
24
|
-
assemblyline_v4_service/updater/helper.py,sha256=
|
|
24
|
+
assemblyline_v4_service/updater/helper.py,sha256=Eo8HNyHRZ3OL_b6onWIFj_BykwYj94JgRrT9GQx2Tsg,10076
|
|
25
25
|
assemblyline_v4_service/updater/updater.py,sha256=HqRVUs5nNIfKmHu7g1GUkgC7bvYWs2EC9ZFy1kj-Ltw,31609
|
|
26
26
|
test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
27
27
|
test/conftest.py,sha256=W3SieQpZsZpGEmtLqY4aIlxREDSsHceyCrFcFsWUM0U,1851
|
|
@@ -38,8 +38,8 @@ test/test_common/test_request.py,sha256=Ceyds8BNO1O0f1kH1VEb84faJcaupvSjVKIrGdHe
|
|
|
38
38
|
test/test_common/test_result.py,sha256=6BiOKxEPrKBjOY44jv3TY-yiXm0qI1ok_CZBnjP9TM4,45447
|
|
39
39
|
test/test_common/test_task.py,sha256=P44mNcSe-3tJgDk9ppN3KbM7oN4LBVIuhONG-Gveh74,19007
|
|
40
40
|
test/test_common/test_utils.py,sha256=TbnBxqpS_ZC5ptXR9XJX3xtbItD0mTbtiBxxdyP8J5k,5904
|
|
41
|
-
assemblyline_v4_service-4.5.1.
|
|
42
|
-
assemblyline_v4_service-4.5.1.
|
|
43
|
-
assemblyline_v4_service-4.5.1.
|
|
44
|
-
assemblyline_v4_service-4.5.1.
|
|
45
|
-
assemblyline_v4_service-4.5.1.
|
|
41
|
+
assemblyline_v4_service-4.5.1.dev436.dist-info/LICENCE.md,sha256=NSkYo9EH8h5oOkzg4VhjAHF4339MqPP2cQ8msTPgl-c,1396
|
|
42
|
+
assemblyline_v4_service-4.5.1.dev436.dist-info/METADATA,sha256=XVQcsdi4RmcV1Bt7KWM_PvG8b288Mda7iOfCtwEFMCQ,5603
|
|
43
|
+
assemblyline_v4_service-4.5.1.dev436.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
|
|
44
|
+
assemblyline_v4_service-4.5.1.dev436.dist-info/top_level.txt,sha256=LpTOEaVCatkrvbVq3EZseMSIa2PQZU-2rhuO_FTpZgY,29
|
|
45
|
+
assemblyline_v4_service-4.5.1.dev436.dist-info/RECORD,,
|
|
@@ -1,276 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.2
|
|
2
|
-
Name: assemblyline-v4-service
|
|
3
|
-
Version: 4.5.1.dev427
|
|
4
|
-
Summary: Assemblyline 4 - Service base
|
|
5
|
-
Home-page: https://github.com/CybercentreCanada/assemblyline-v4-service/
|
|
6
|
-
Author: CCCS Assemblyline development team
|
|
7
|
-
Author-email: assemblyline@cyber.gc.ca
|
|
8
|
-
License: MIT
|
|
9
|
-
Keywords: assemblyline automated malware analysis gc canada cse-cst cse cst cyber cccs
|
|
10
|
-
Classifier: Development Status :: 5 - Production/Stable
|
|
11
|
-
Classifier: Intended Audience :: Developers
|
|
12
|
-
Classifier: Topic :: Software Development :: Libraries
|
|
13
|
-
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
-
Classifier: Programming Language :: Python :: 3.7
|
|
15
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
16
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
-
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
-
Description-Content-Type: text/markdown
|
|
21
|
-
License-File: LICENCE.md
|
|
22
|
-
Requires-Dist: assemblyline
|
|
23
|
-
Requires-Dist: assemblyline-core
|
|
24
|
-
Requires-Dist: cart
|
|
25
|
-
Requires-Dist: fuzzywuzzy
|
|
26
|
-
Requires-Dist: pefile
|
|
27
|
-
Requires-Dist: pillow!=10.1.0,!=10.2.0
|
|
28
|
-
Requires-Dist: python-Levenshtein
|
|
29
|
-
Requires-Dist: regex
|
|
30
|
-
Provides-Extra: updater
|
|
31
|
-
Requires-Dist: gunicorn[gevent]; extra == "updater"
|
|
32
|
-
Requires-Dist: flask; extra == "updater"
|
|
33
|
-
Requires-Dist: gitpython; extra == "updater"
|
|
34
|
-
Requires-Dist: git-remote-codecommit; extra == "updater"
|
|
35
|
-
Requires-Dist: psutil; extra == "updater"
|
|
36
|
-
Dynamic: author
|
|
37
|
-
Dynamic: author-email
|
|
38
|
-
Dynamic: classifier
|
|
39
|
-
Dynamic: description
|
|
40
|
-
Dynamic: description-content-type
|
|
41
|
-
Dynamic: home-page
|
|
42
|
-
Dynamic: keywords
|
|
43
|
-
Dynamic: license
|
|
44
|
-
Dynamic: provides-extra
|
|
45
|
-
Dynamic: requires-dist
|
|
46
|
-
Dynamic: summary
|
|
47
|
-
|
|
48
|
-
# Assemblyline 4 - Service Base
|
|
49
|
-
|
|
50
|
-
This repository provides the base service functionality for Assemblyline 4 services.
|
|
51
|
-
|
|
52
|
-
## Creating a new Assemblyline service
|
|
53
|
-
|
|
54
|
-
### Service file structure
|
|
55
|
-
|
|
56
|
-
An Assemblyline service has the following file structure:
|
|
57
|
-
|
|
58
|
-
```text
|
|
59
|
-
assemblyline-service-<service name>
|
|
60
|
-
│
|
|
61
|
-
├── Dockerfile
|
|
62
|
-
├── <service name>.py
|
|
63
|
-
└── service_manifest.yml
|
|
64
|
-
```
|
|
65
|
-
|
|
66
|
-
This is overview of what each of these does:
|
|
67
|
-
|
|
68
|
-
- `Dockerfile` ─ Build file for the service container, see _Dockerfile_ section below for more details
|
|
69
|
-
- `<service name>.py` ─ Contains main service code
|
|
70
|
-
- `service_manifest.yml` ─ Service definition file, see _Service manifest_ section below for more details
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
### Service manifest
|
|
74
|
-
|
|
75
|
-
Every service must have a `service_manifest.yml` file in its root directory. The manifest file presents essential information about the service to the Assemblyline core system, information the system must have before it can run the service.
|
|
76
|
-
|
|
77
|
-
The diagram below shows all the elements that the manifest file can contain, including a brief description of each.
|
|
78
|
-
|
|
79
|
-
```yaml
|
|
80
|
-
# Name of the service
|
|
81
|
-
name: ResultSample
|
|
82
|
-
# Version of the service
|
|
83
|
-
version: 1
|
|
84
|
-
# Description of the service
|
|
85
|
-
description: >
|
|
86
|
-
ALv4 Result example service
|
|
87
|
-
|
|
88
|
-
This service provides examples of how to:
|
|
89
|
-
- define your service manifest
|
|
90
|
-
- use the different section types
|
|
91
|
-
- use tags
|
|
92
|
-
- use heuristics to score sections
|
|
93
|
-
- use the att&ck matrix
|
|
94
|
-
- use the updater framework
|
|
95
|
-
- define submission parameters
|
|
96
|
-
- define service configuration parameters
|
|
97
|
-
|
|
98
|
-
# Regex defining the types of files the service accepts and rejects
|
|
99
|
-
accepts: .*
|
|
100
|
-
rejects: empty|metadata/.*
|
|
101
|
-
|
|
102
|
-
# At which stage the service should run (one of: FILTER, EXTRACT, CORE, SECONDARY, POST)
|
|
103
|
-
# NOTE: Stages are executed in the order defined in the list
|
|
104
|
-
stage: CORE
|
|
105
|
-
# Which category the service is part of (one of: Antivirus, Dynamic Analysis, External, Extraction, Filtering, Networking, Static Analysis)
|
|
106
|
-
category: Static Analysis
|
|
107
|
-
|
|
108
|
-
# Does the service require access to the file to perform its task
|
|
109
|
-
# If set to false, the service will only have access to the file metadata (e.g. Hashes, size, type, ...)
|
|
110
|
-
file_required: true
|
|
111
|
-
# Maximum execution time the service has before it's considered to be timed out
|
|
112
|
-
timeout: 60
|
|
113
|
-
# Does the service force the caching of results to be disabled
|
|
114
|
-
# (only use for service that will always provided different results each run)
|
|
115
|
-
disable_cache: false
|
|
116
|
-
|
|
117
|
-
# is the service enabled by default
|
|
118
|
-
enabled: true
|
|
119
|
-
# does the service make APIs call to other product not part of the assemblyline infrastructure (e.g. VirusTotal, ...)
|
|
120
|
-
is_external: false
|
|
121
|
-
# Number of concurrent services allowed to run at the same time
|
|
122
|
-
licence_count: 0
|
|
123
|
-
|
|
124
|
-
# service configuration block (dictionary of config variables)
|
|
125
|
-
# NOTE: The key names can be anything and the value can be of any types
|
|
126
|
-
config:
|
|
127
|
-
str_config: value1
|
|
128
|
-
int_config: 1
|
|
129
|
-
list_config: [1, 2, 3, 4]
|
|
130
|
-
bool_config: false
|
|
131
|
-
|
|
132
|
-
# submission params block: a list of submission param object that define parameters
|
|
133
|
-
# that the user can change about the service for each of its scans
|
|
134
|
-
# supported types: bool, int, str, list
|
|
135
|
-
submission_params:
|
|
136
|
-
- default: ""
|
|
137
|
-
name: password
|
|
138
|
-
type: str
|
|
139
|
-
value: ""
|
|
140
|
-
- default: false
|
|
141
|
-
name: extra_work
|
|
142
|
-
type: bool
|
|
143
|
-
value: false
|
|
144
|
-
|
|
145
|
-
# Service heuristic blocks: List of heuristics object that define the different heuristics used in the service
|
|
146
|
-
heuristics:
|
|
147
|
-
- description: This the first Heuristic for ResultSample service.
|
|
148
|
-
filetype: pdf
|
|
149
|
-
heur_id: AL_RESULTSAMPLE_1
|
|
150
|
-
name: Masks has PDF
|
|
151
|
-
score: 100
|
|
152
|
-
attack_id: T1001
|
|
153
|
-
- description: This is second Heuristic for ResultSample service.
|
|
154
|
-
filetype: exe
|
|
155
|
-
heur_id: AL_RESULTSAMPLE_2
|
|
156
|
-
name: Drops an exe
|
|
157
|
-
score: 1000
|
|
158
|
-
- description: This is third Heuristic for ResultSample service.
|
|
159
|
-
filetype: exe
|
|
160
|
-
heur_id: AL_RESULTSAMPLE_3
|
|
161
|
-
name: Extraction information
|
|
162
|
-
score: 0
|
|
163
|
-
|
|
164
|
-
# Docker configuration block which defines:
|
|
165
|
-
# - the name of the docker container that will be created
|
|
166
|
-
# - cpu and ram allocation by the container
|
|
167
|
-
docker_config:
|
|
168
|
-
image: cccs/assemblyline-service-resultsample:latest
|
|
169
|
-
cpu_cores: 1.0
|
|
170
|
-
ram_mb_min: 128
|
|
171
|
-
ram_mb: 256
|
|
172
|
-
|
|
173
|
-
# Update configuration block
|
|
174
|
-
update_config:
|
|
175
|
-
# list of source object from where to fetch files for update and what will be the name of those files on disk
|
|
176
|
-
sources:
|
|
177
|
-
- uri: https://file-examples.com/wp-content/uploads/2017/02/zip_2MB.zip
|
|
178
|
-
name: sample_2mb_file
|
|
179
|
-
- uri: https://file-examples.com/wp-content/uploads/2017/02/zip_5MB.zip
|
|
180
|
-
name: sample_5mb_file
|
|
181
|
-
# intervale in seconds at which the updater runs
|
|
182
|
-
update_interval_seconds: 300
|
|
183
|
-
# Should the downloaded files be used to create signatures in the system
|
|
184
|
-
generates_signatures: false
|
|
185
|
-
```
|
|
186
|
-
|
|
187
|
-
### Dockerfile
|
|
188
|
-
|
|
189
|
-
A Dockerfile is required to build the service container that will be executed in the system.
|
|
190
|
-
|
|
191
|
-
The following items must be set for all services:
|
|
192
|
-
|
|
193
|
-
- All services must be based on the `cccs/assemblyline-v4-service-base:latest` image
|
|
194
|
-
- An environment variable must be set for the service path
|
|
195
|
-
- Install any service requirements
|
|
196
|
-
- Copy the service code into `/opt/al/al_service/`
|
|
197
|
-
|
|
198
|
-
```dockerfile
|
|
199
|
-
FROM cccs/assemblyline-v4-service-base:latest
|
|
200
|
-
|
|
201
|
-
# Set the service path
|
|
202
|
-
ENV SERVICE_PATH result_sample.ResultSample
|
|
203
|
-
|
|
204
|
-
# By default, the base service container as the assemblyline user as the running user
|
|
205
|
-
# switch to root to perform installation of dependancies
|
|
206
|
-
USER root
|
|
207
|
-
|
|
208
|
-
# See that we all these operations in one line to reduce
|
|
209
|
-
# the number of container layers and size of the container
|
|
210
|
-
RUN apt-get update && apt-get install -y my_debian_apt_dependency_package && rm -rf /var/lib/apt/lists/*
|
|
211
|
-
RUN pip install --no-cache-dir --user my_pip_dependency && rm -rf ~/.cache/pip
|
|
212
|
-
|
|
213
|
-
# Change to the assemblyline user to make sure your service does not run as root
|
|
214
|
-
USER assemblyline
|
|
215
|
-
|
|
216
|
-
# Copy the service code in the service directory
|
|
217
|
-
WORKDIR /opt/al_service
|
|
218
|
-
COPY assemblyline_result_sample_service .
|
|
219
|
-
```
|
|
220
|
-
|
|
221
|
-
## Testing an Assemblyline service
|
|
222
|
-
|
|
223
|
-
To test an Assemblyline service in standalone mode, the [run_service_once.py](https://github.com/CybercentreCanada/assemblyline-v4-service/src/master/dev/run_service_once.py) script can be used to run a single task through the service for testing. That script does not require that you have a working version of Assemblyline installed, all you need are the Assemblyline python libraries.
|
|
224
|
-
|
|
225
|
-
### Setting up dev environment
|
|
226
|
-
|
|
227
|
-
**NOTE:** The following environment setup has only been tested on Ubuntu 20.04.
|
|
228
|
-
|
|
229
|
-
1. Install required packages
|
|
230
|
-
|
|
231
|
-
```
|
|
232
|
-
sudo apt-get install build-essential libffi-dev python3.7 python3.7-dev python3-pip automake autoconf libtool
|
|
233
|
-
```
|
|
234
|
-
|
|
235
|
-
2. Install Assemblyline v4 service package
|
|
236
|
-
|
|
237
|
-
```
|
|
238
|
-
pip install --no-cache-dir --user assemblyline-v4-service
|
|
239
|
-
```
|
|
240
|
-
|
|
241
|
-
3. Add your service development directory path (ie. `/home/ubuntu/assemblyline-v4-service`) to the PYTHONPATH environment variable
|
|
242
|
-
|
|
243
|
-
### Using the `run_service_once.py` script
|
|
244
|
-
|
|
245
|
-
#### Steps
|
|
246
|
-
|
|
247
|
-
1. Ensure the current working directory is the root of the service directory of the service to be run
|
|
248
|
-
|
|
249
|
-
```shell
|
|
250
|
-
cd assemblyline-service-<service name>
|
|
251
|
-
```
|
|
252
|
-
|
|
253
|
-
2. From a terminal, run the `run_service_once` script, where `<service path>` is the path to the service module and `<file path>` is the path of the file to be processed
|
|
254
|
-
|
|
255
|
-
```shell
|
|
256
|
-
python3.11 -m assemblyline_v4_service.dev.run_service_once <service path> <file path>
|
|
257
|
-
```
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
3. The output of the service (`result.json` and extracted/supplementary files) will be located in a directory where the input file is located
|
|
261
|
-
|
|
262
|
-
#### Example of running the ResultSample service
|
|
263
|
-
|
|
264
|
-
1. Change working directory to root of the service:
|
|
265
|
-
|
|
266
|
-
```shell
|
|
267
|
-
cd assemblyline_result_sample_service
|
|
268
|
-
```
|
|
269
|
-
|
|
270
|
-
2. From a terminal, run the `run_service_once` script
|
|
271
|
-
|
|
272
|
-
```shell
|
|
273
|
-
python3.11 -m assemblyline_v4_service.dev.run_service_once assemblyline_result_sample_service.result_sample.ResultSample /home/ubuntu/testfile.doc
|
|
274
|
-
```
|
|
275
|
-
|
|
276
|
-
3. The `results.json` and any extracted/supplementary files will be outputted to `/home/ubuntu/testfile_resultsample`
|
|
File without changes
|
|
File without changes
|
|
File without changes
|