assemblyline-core 4.5.1.dev87__tar.gz → 4.5.1.dev89__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of assemblyline-core might be problematic. Click here for more details.

Files changed (88) hide show
  1. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/PKG-INFO +1 -1
  2. assemblyline-core-4.5.1.dev89/assemblyline_core/VERSION +1 -0
  3. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/metrics/heartbeat_formatter.py +1 -0
  4. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/metrics/metrics_server.py +13 -7
  5. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core.egg-info/PKG-INFO +1 -1
  6. assemblyline-core-4.5.1.dev87/assemblyline_core/VERSION +0 -1
  7. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/LICENCE.md +0 -0
  8. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/README.md +0 -0
  9. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/__init__.py +0 -0
  10. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/alerter/__init__.py +0 -0
  11. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/alerter/processing.py +0 -0
  12. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/alerter/run_alerter.py +0 -0
  13. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/archiver/__init__.py +0 -0
  14. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/archiver/run_archiver.py +0 -0
  15. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/badlist_client.py +0 -0
  16. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/dispatching/__init__.py +0 -0
  17. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/dispatching/__main__.py +0 -0
  18. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/dispatching/client.py +0 -0
  19. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/dispatching/dispatcher.py +0 -0
  20. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/dispatching/schedules.py +0 -0
  21. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/dispatching/timeout.py +0 -0
  22. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/expiry/__init__.py +0 -0
  23. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/expiry/run_expiry.py +0 -0
  24. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/ingester/__init__.py +0 -0
  25. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/ingester/__main__.py +0 -0
  26. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/ingester/constants.py +0 -0
  27. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/ingester/ingester.py +0 -0
  28. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/metrics/__init__.py +0 -0
  29. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/metrics/es_metrics.py +0 -0
  30. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/metrics/helper.py +0 -0
  31. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/metrics/run_heartbeat_manager.py +0 -0
  32. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/metrics/run_metrics_aggregator.py +0 -0
  33. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/metrics/run_statistics_aggregator.py +0 -0
  34. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/plumber/__init__.py +0 -0
  35. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/plumber/run_plumber.py +0 -0
  36. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/replay/__init__.py +0 -0
  37. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/replay/client.py +0 -0
  38. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/replay/creator/__init__.py +0 -0
  39. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/replay/creator/run.py +0 -0
  40. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/replay/creator/run_worker.py +0 -0
  41. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/replay/loader/__init__.py +0 -0
  42. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/replay/loader/run.py +0 -0
  43. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/replay/loader/run_worker.py +0 -0
  44. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/replay/replay.py +0 -0
  45. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/safelist_client.py +0 -0
  46. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/scaler/__init__.py +0 -0
  47. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/scaler/collection.py +0 -0
  48. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/scaler/controllers/__init__.py +0 -0
  49. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/scaler/controllers/docker_ctl.py +0 -0
  50. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/scaler/controllers/interface.py +0 -0
  51. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/scaler/controllers/kubernetes_ctl.py +0 -0
  52. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/scaler/run_scaler.py +0 -0
  53. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/scaler/scaler_server.py +0 -0
  54. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/server_base.py +0 -0
  55. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/signature_client.py +0 -0
  56. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/submission_client.py +0 -0
  57. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/tasking_client.py +0 -0
  58. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/updater/__init__.py +0 -0
  59. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/updater/helper.py +0 -0
  60. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/updater/run_updater.py +0 -0
  61. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/vacuum/__init__.py +0 -0
  62. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/vacuum/crawler.py +0 -0
  63. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/vacuum/department_map.py +0 -0
  64. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/vacuum/safelist.py +0 -0
  65. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/vacuum/stream_map.py +0 -0
  66. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/vacuum/worker.py +0 -0
  67. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/workflow/__init__.py +0 -0
  68. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/workflow/run_workflow.py +0 -0
  69. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core.egg-info/SOURCES.txt +0 -0
  70. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core.egg-info/dependency_links.txt +0 -0
  71. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core.egg-info/requires.txt +0 -0
  72. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core.egg-info/top_level.txt +0 -0
  73. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/setup.cfg +0 -0
  74. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/setup.py +0 -0
  75. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/test/test_alerter.py +0 -0
  76. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/test/test_badlist_client.py +0 -0
  77. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/test/test_dispatcher.py +0 -0
  78. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/test/test_expiry.py +0 -0
  79. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/test/test_plumber.py +0 -0
  80. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/test/test_replay.py +0 -0
  81. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/test/test_safelist_client.py +0 -0
  82. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/test/test_scaler.py +0 -0
  83. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/test/test_scheduler.py +0 -0
  84. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/test/test_signature_client.py +0 -0
  85. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/test/test_simulation.py +0 -0
  86. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/test/test_vacuum.py +0 -0
  87. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/test/test_worker_ingest.py +0 -0
  88. {assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/test/test_worker_submit.py +0 -0
{assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: assemblyline-core
3
- Version: 4.5.1.dev87
3
+ Version: 4.5.1.dev89
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
assemblyline-core-4.5.1.dev89/assemblyline_core/VERSION ADDED
@@ -0,0 +1 @@
1
+ 4.5.1.dev89
{assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/metrics/heartbeat_formatter.py RENAMED
@@ -297,6 +297,7 @@ class HeartbeatFormatter(object):
297
297
  "sender": self.sender,
298
298
  "msg": {
299
299
  "instances": instances,
300
+ 'unassigned_shards': m_data['unassigned'],
300
301
  "request_time": m_data['request_time'],
301
302
  "shard_sizes": [{'name': index, 'shard_size': size}
302
303
  for index, size in m_data['shard_sizes'].items()],
{assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core/metrics/metrics_server.py RENAMED
@@ -335,21 +335,26 @@ class HeartbeatManager(ServerBase):
335
335
  elapsed = time.time() - send_time
336
336
  remaining = self.config.core.metrics.export_interval - elapsed
337
337
  if remaining > 0:
338
- self.sleep(remaining)
338
+ self.sleep(remaining)
339
339
 
340
340
  def _fetch_shards(self):
341
341
  request_time = None
342
342
  sizes = {}
343
- nodes = []
343
+ unassigned = 0
344
+ nodes = set()
344
345
  try:
345
346
  # Pull shard data from elastisearch
346
347
  start_time = time.time()
347
348
  response = self.datastore.ds.client.cat.shards(bytes='b', format='json', master_timeout='5s')
348
349
  for shard in response.body:
349
350
  index = shard['index']
350
- sizes.setdefault(index, 0)
351
- sizes[index] = max(sizes[index], int(shard['store']))
352
- nodes.append(shard['node'])
351
+ node = shard['node']
352
+ if node:
353
+ sizes.setdefault(index, 0)
354
+ sizes[index] = max(sizes[index], int(shard.get('store') or 0))
355
+ nodes.add(node)
356
+ else:
357
+ unassigned += 1
353
358
  request_time = time.time() - start_time
354
359
 
355
360
  finally:
@@ -358,8 +363,9 @@ class HeartbeatManager(ServerBase):
358
363
  metrics = {
359
364
  'shard_sizes': sizes,
360
365
  'request_time': request_time,
366
+ 'unassigned': unassigned
361
367
  }
362
- self.hm.send_heartbeat('elastic', 'datastore', metrics, len(set(nodes)))
368
+ self.hm.send_heartbeat('elastic', 'datastore', metrics, len(nodes))
363
369
 
364
370
  def _fetch_retrohunt(self):
365
371
  status = {}
@@ -370,7 +376,7 @@ class HeartbeatManager(ServerBase):
370
376
  status = self.hauntedhouse_client.status()
371
377
  request_time = time.time() - start_time
372
378
 
373
- finally:
379
+ finally:
374
380
  metrics = {
375
381
  'request_time': request_time,
376
382
  'status': status,
{assemblyline-core-4.5.1.dev87 → assemblyline-core-4.5.1.dev89}/assemblyline_core.egg-info/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: assemblyline-core
3
- Version: 4.5.1.dev87
3
+ Version: 4.5.1.dev89
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
assemblyline-core-4.5.1.dev87/assemblyline_core/VERSION DELETED
@@ -1 +0,0 @@
1
- 4.5.1.dev87