assemblyline-core 4.4.1.dev175__tar.gz → 4.4.1.dev176__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of assemblyline-core might be problematic. Click here for more details.

Files changed (83) hide show
  1. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/PKG-INFO +1 -1
  2. assemblyline-core-4.4.1.dev176/assemblyline_core/VERSION +1 -0
  3. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/workflow/run_workflow.py +10 -8
  4. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core.egg-info/PKG-INFO +1 -1
  5. assemblyline-core-4.4.1.dev175/assemblyline_core/VERSION +0 -1
  6. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/LICENCE.md +0 -0
  7. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/README.md +0 -0
  8. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/__init__.py +0 -0
  9. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/alerter/__init__.py +0 -0
  10. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/alerter/processing.py +0 -0
  11. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/alerter/run_alerter.py +0 -0
  12. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/archiver/__init__.py +0 -0
  13. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/archiver/run_archiver.py +0 -0
  14. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/dispatching/__init__.py +0 -0
  15. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/dispatching/__main__.py +0 -0
  16. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/dispatching/client.py +0 -0
  17. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/dispatching/dispatcher.py +0 -0
  18. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/dispatching/schedules.py +0 -0
  19. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/dispatching/timeout.py +0 -0
  20. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/expiry/__init__.py +0 -0
  21. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/expiry/run_expiry.py +0 -0
  22. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/ingester/__init__.py +0 -0
  23. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/ingester/__main__.py +0 -0
  24. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/ingester/constants.py +0 -0
  25. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/ingester/ingester.py +0 -0
  26. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/metrics/__init__.py +0 -0
  27. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/metrics/es_metrics.py +0 -0
  28. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/metrics/heartbeat_formatter.py +0 -0
  29. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/metrics/helper.py +0 -0
  30. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/metrics/metrics_server.py +0 -0
  31. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/metrics/run_heartbeat_manager.py +0 -0
  32. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/metrics/run_metrics_aggregator.py +0 -0
  33. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/metrics/run_statistics_aggregator.py +0 -0
  34. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/plumber/__init__.py +0 -0
  35. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/plumber/run_plumber.py +0 -0
  36. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/replay/__init__.py +0 -0
  37. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/replay/client.py +0 -0
  38. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/replay/creator/__init__.py +0 -0
  39. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/replay/creator/run.py +0 -0
  40. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/replay/creator/run_worker.py +0 -0
  41. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/replay/loader/__init__.py +0 -0
  42. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/replay/loader/run.py +0 -0
  43. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/replay/loader/run_worker.py +0 -0
  44. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/replay/replay.py +0 -0
  45. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/safelist_client.py +0 -0
  46. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/scaler/__init__.py +0 -0
  47. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/scaler/collection.py +0 -0
  48. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/scaler/controllers/__init__.py +0 -0
  49. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/scaler/controllers/docker_ctl.py +0 -0
  50. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/scaler/controllers/interface.py +0 -0
  51. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/scaler/controllers/kubernetes_ctl.py +0 -0
  52. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/scaler/run_scaler.py +0 -0
  53. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/scaler/scaler_server.py +0 -0
  54. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/server_base.py +0 -0
  55. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/submission_client.py +0 -0
  56. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/tasking_client.py +0 -0
  57. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/updater/__init__.py +0 -0
  58. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/updater/helper.py +0 -0
  59. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/updater/run_updater.py +0 -0
  60. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/vacuum/__init__.py +0 -0
  61. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/vacuum/crawler.py +0 -0
  62. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/vacuum/department_map.py +0 -0
  63. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/vacuum/safelist.py +0 -0
  64. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/vacuum/stream_map.py +0 -0
  65. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/vacuum/worker.py +0 -0
  66. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core/workflow/__init__.py +0 -0
  67. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core.egg-info/SOURCES.txt +0 -0
  68. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core.egg-info/dependency_links.txt +0 -0
  69. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core.egg-info/requires.txt +0 -0
  70. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/assemblyline_core.egg-info/top_level.txt +0 -0
  71. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/setup.cfg +0 -0
  72. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/setup.py +0 -0
  73. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/test/test_alerter.py +0 -0
  74. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/test/test_dispatcher.py +0 -0
  75. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/test/test_expiry.py +0 -0
  76. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/test/test_plumber.py +0 -0
  77. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/test/test_replay.py +0 -0
  78. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/test/test_scaler.py +0 -0
  79. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/test/test_scheduler.py +0 -0
  80. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/test/test_simulation.py +0 -0
  81. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/test/test_vacuum.py +0 -0
  82. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/test/test_worker_ingest.py +0 -0
  83. {assemblyline-core-4.4.1.dev175 → assemblyline-core-4.4.1.dev176}/test/test_worker_submit.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: assemblyline-core
3
- Version: 4.4.1.dev175
3
+ Version: 4.4.1.dev176
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -0,0 +1 @@
1
+ 4.4.1.dev176
@@ -38,7 +38,7 @@ class WorkflowManager(ServerBase):
38
38
  if self.apm_client:
39
39
  self.apm_client.begin_transaction("Get last reporting timestamp")
40
40
 
41
- self.log.info("Finding reporting timestamp for the last alert since {start_ts}...".format(start_ts=p_start_ts))
41
+ self.log.info(f"Finding reporting timestamp for the last alert since {p_start_ts}...")
42
42
  result = None
43
43
  while result is None:
44
44
  try:
@@ -125,16 +125,18 @@ class WorkflowManager(ServerBase):
125
125
  self.apm_client.end_transaction(workflow.name, 'no_action')
126
126
  continue
127
127
 
128
- fq = ["reporting_ts:[{start_ts} TO {end_ts}]".format(start_ts=self.start_ts, end_ts=end_ts)]
128
+ fq = [f"reporting_ts:[{self.start_ts} TO {end_ts}]", "NOT extended_scan:submitted"]
129
129
 
130
- event_data = Event({'entity_type': 'workflow', 'entity_id': workflow.workflow_id, 'entity_name': workflow.name})
130
+ event_data = Event({'entity_type': 'workflow',
131
+ 'entity_id': workflow.workflow_id,
132
+ 'entity_name': workflow.name})
131
133
  operations = []
132
134
  fq_items = []
133
135
  if labels:
134
136
  operations.extend([(self.datastore.alert.UPDATE_APPEND_IF_MISSING, 'label', lbl)
135
137
  for lbl in labels])
136
138
  for label in labels:
137
- fq_items.append("label:\"{label}\"".format(label=label))
139
+ fq_items.append(f'label:"{label}"')
138
140
  event_data.labels = labels
139
141
  if priority:
140
142
  operations.append((self.datastore.alert.UPDATE_SET, 'priority', priority))
@@ -145,7 +147,7 @@ class WorkflowManager(ServerBase):
145
147
  fq_items.append("(status:MALICIOUS OR status:NON-MALICIOUS OR status:ASSESS)")
146
148
  event_data.status = status
147
149
 
148
- fq.append("NOT ({exclusion})".format(exclusion=" AND ".join(fq_items)))
150
+ fq.append(f"NOT ({' AND '.join(fq_items)})")
149
151
  # Add event to alert's audit history
150
152
  operations.append((self.datastore.alert.UPDATE_APPEND, 'events', event_data))
151
153
 
@@ -155,7 +157,7 @@ class WorkflowManager(ServerBase):
155
157
  elasticapm.label(affected_alerts=count)
156
158
 
157
159
  if count:
158
- self.log.info("{count} Alert(s) were affected by this filter.".format(count=count))
160
+ self.log.info(f"{count} Alert(s) were affected by this filter.")
159
161
  if workflow.workflow_id != "DEFAULT":
160
162
  seen = now_as_iso()
161
163
  operations = [
@@ -187,7 +189,7 @@ class WorkflowManager(ServerBase):
187
189
  self.apm_client.begin_transaction("Mark alerts complete")
188
190
 
189
191
  self.log.info(f'Marking all alerts between {self.start_ts} and {end_ts} as workflow completed...')
190
- wc_query = "reporting_ts:[{start_ts} TO {end_ts}]".format(start_ts=self.start_ts, end_ts=end_ts)
192
+ wc_query = f"reporting_ts:[{self.start_ts} TO {end_ts}]"
191
193
  wc_operations = [(self.datastore.alert.UPDATE_SET, 'workflows_completed', True)]
192
194
  try:
193
195
  wc_count = self.datastore.alert.update_by_query(wc_query, wc_operations)
@@ -195,7 +197,7 @@ class WorkflowManager(ServerBase):
195
197
  elasticapm.label(affected_alerts=wc_count)
196
198
 
197
199
  if wc_count:
198
- self.log.info("{count} Alert(s) workflows marked as completed.".format(count=count))
200
+ self.log.info(f"{count} Alert(s) workflows marked as completed.")
199
201
 
200
202
  # End of transaction
201
203
  if self.apm_client:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: assemblyline-core
3
- Version: 4.4.1.dev175
3
+ Version: 4.4.1.dev176
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -1 +0,0 @@
1
- 4.4.1.dev175