assemblyline-core 4.7.5.dev1__tar.gz → 4.7.5.dev2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/PKG-INFO +1 -1
  2. assemblyline_core-4.7.5.dev2/assemblyline_core/VERSION +1 -0
  3. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/updater/run_updater.py +24 -17
  4. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core.egg-info/PKG-INFO +1 -1
  5. assemblyline_core-4.7.5.dev1/assemblyline_core/VERSION +0 -1
  6. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/LICENCE.md +0 -0
  7. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/README.md +0 -0
  8. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/__init__.py +0 -0
  9. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/alerter/__init__.py +0 -0
  10. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/alerter/processing.py +0 -0
  11. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/alerter/run_alerter.py +0 -0
  12. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/archiver/__init__.py +0 -0
  13. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/archiver/run_archiver.py +0 -0
  14. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/badlist_client.py +0 -0
  15. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/dispatching/__init__.py +0 -0
  16. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/dispatching/client.py +0 -0
  17. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/dispatching/dispatcher.py +0 -0
  18. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/dispatching/schedules.py +0 -0
  19. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/expiry/__init__.py +0 -0
  20. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/expiry/run_expiry.py +0 -0
  21. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/ingester/__init__.py +0 -0
  22. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/ingester/constants.py +0 -0
  23. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/ingester/ingester.py +0 -0
  24. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/metrics/__init__.py +0 -0
  25. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/metrics/es_metrics.py +0 -0
  26. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/metrics/heartbeat_formatter.py +0 -0
  27. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/metrics/helper.py +0 -0
  28. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/metrics/metrics_server.py +0 -0
  29. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/metrics/run_heartbeat_manager.py +0 -0
  30. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/metrics/run_metrics_aggregator.py +0 -0
  31. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/metrics/run_statistics_aggregator.py +0 -0
  32. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/replay/__init__.py +0 -0
  33. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/replay/client.py +0 -0
  34. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/replay/creator/__init__.py +0 -0
  35. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/replay/creator/run.py +0 -0
  36. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/replay/creator/run_worker.py +0 -0
  37. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/replay/loader/__init__.py +0 -0
  38. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/replay/loader/run.py +0 -0
  39. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/replay/loader/run_worker.py +0 -0
  40. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/replay/replay.py +0 -0
  41. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/safelist_client.py +0 -0
  42. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/scaler/__init__.py +0 -0
  43. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/scaler/collection.py +0 -0
  44. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/scaler/controllers/__init__.py +0 -0
  45. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/scaler/controllers/docker_ctl.py +0 -0
  46. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/scaler/controllers/interface.py +0 -0
  47. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/scaler/controllers/kubernetes_ctl.py +0 -0
  48. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/scaler/run_scaler.py +0 -0
  49. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/scaler/scaler_server.py +0 -0
  50. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/server_base.py +0 -0
  51. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/signature_client.py +0 -0
  52. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/submission_client.py +0 -0
  53. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/tasking_client.py +0 -0
  54. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/updater/__init__.py +0 -0
  55. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/updater/helper.py +0 -0
  56. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/vacuum/__init__.py +0 -0
  57. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/vacuum/crawler.py +0 -0
  58. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/vacuum/department_map.py +0 -0
  59. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/vacuum/safelist.py +0 -0
  60. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/vacuum/stream_map.py +0 -0
  61. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/vacuum/worker.py +0 -0
  62. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/workflow/__init__.py +0 -0
  63. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/workflow/run_workflow.py +0 -0
  64. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core.egg-info/SOURCES.txt +0 -0
  65. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core.egg-info/dependency_links.txt +0 -0
  66. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core.egg-info/requires.txt +0 -0
  67. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core.egg-info/top_level.txt +0 -0
  68. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/setup.cfg +0 -0
  69. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/setup.py +0 -0
  70. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/test/test_alerter.py +0 -0
  71. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/test/test_badlist_client.py +0 -0
  72. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/test/test_expiry.py +0 -0
  73. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/test/test_replay.py +0 -0
  74. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/test/test_safelist_client.py +0 -0
  75. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/test/test_scaler.py +0 -0
  76. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/test/test_scheduler.py +0 -0
  77. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/test/test_signature_client.py +0 -0
  78. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/test/test_tasking_client.py +0 -0
  79. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/test/test_vacuum.py +0 -0
  80. {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/test/test_workflow.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: assemblyline-core
3
- Version: 4.7.5.dev1
3
+ Version: 4.7.5.dev2
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -0,0 +1 @@
1
+ 4.7.5.dev2
@@ -489,12 +489,18 @@ class ServiceUpdater(ThreadedCoreBase):
489
489
  # Add any mounts defined in the service defaults to the updater interface so they get applied to all updates/installs
490
490
  self.mounts = self.config.core.scaler.service_defaults.mounts
491
491
 
492
+ # If there are no services installed push the auto install services to the install table
493
+ installed_service_names = list(self.datastore.service_delta.keys())
494
+ if len(installed_service_names) == 0:
495
+ for service in self.config.core.updater.auto_install:
496
+ self.container_install.add(service.name, {'image': service.image})
497
+
492
498
  def _handle_service_change_event(self, data: Optional[ServiceChange]):
493
499
  if data is not None:
494
500
  if data.operation == Operation.Incompatible:
495
501
  self.incompatible_services.add(data.name)
496
502
 
497
- def container_installs(self):
503
+ def container_installs(self) -> None:
498
504
  """Go through the list of services and check what are the latest tags for it"""
499
505
  while self.running:
500
506
  self.log.info("[CI] Installing all services marked for install...")
@@ -561,8 +567,8 @@ class ServiceUpdater(ThreadedCoreBase):
561
567
  )
562
568
 
563
569
  except Exception as e:
564
- self.log.error(
565
- f"[CI] Service {service_name} has failed to install. Install procedure cancelled... [{str(e)}]")
570
+ self.log.error("[CI] Service %s has failed to install. Install procedure cancelled... [%s]",
571
+ service_name, e)
566
572
  finally:
567
573
  self.container_keys.pop(api_key)
568
574
 
@@ -574,17 +580,18 @@ class ServiceUpdater(ThreadedCoreBase):
574
580
  for service_name, install_data in self.container_install.items().items():
575
581
  install_threads.append(service_installs_exec.submit(install_service, service_name, install_data))
576
582
 
577
- # Check the status of the container installs
578
- while install_threads:
579
- # Delay each check by 5 seconds to give the container adequate time to perform version registration
580
- time.sleep(5)
581
- pending_threads = []
582
- for thread in install_threads:
583
- if not thread.done():
584
- # Add thead to the list of still pending jobs
585
- pending_threads.append(thread)
586
- else:
587
- service_key = thread.result()
583
+ # Check the status of the container installs
584
+ while install_threads:
585
+ # Delay each check by 5 seconds to give the container adequate time to perform version registration
586
+ time.sleep(5)
587
+ pending_threads = []
588
+ for thread in install_threads:
589
+ if not thread.done():
590
+ # Add thead to the list of still pending jobs
591
+ pending_threads.append(thread)
592
+ continue
593
+
594
+ service_key: str = thread.result()
588
595
  service_name, latest_tag = service_key.split("_")
589
596
 
590
597
  if self.datastore.service.get_if_exists(service_key):
@@ -612,13 +619,13 @@ class ServiceUpdater(ThreadedCoreBase):
612
619
  f"service key ({service_key}) does not exist. Install procedure cancelled...")
613
620
  self.container_install.pop(service_name)
614
621
 
615
- # Update with still pending and loop until all are completed/failed
616
- install_threads = pending_threads
622
+ # Update with still pending and loop until all are completed/failed
623
+ install_threads = pending_threads
617
624
 
618
625
  # Clear out any old dead containers
619
626
  self.controller.cleanup_stale()
620
627
 
621
- self.log.info(f"[CI] Done installing services, waiting {UPDATE_CHECK_INTERVAL} seconds for next install...")
628
+ self.log.info("[CI] Done installing services, waiting {UPDATE_CHECK_INTERVAL} seconds for next install...")
622
629
  time.sleep(UPDATE_CHECK_INTERVAL)
623
630
 
624
631
  def container_updates(self):
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: assemblyline-core
3
- Version: 4.7.5.dev1
3
+ Version: 4.7.5.dev2
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -1 +0,0 @@
1
- 4.7.5.dev1