assemblyline-core 4.7.5.dev1__tar.gz → 4.7.5.dev2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/PKG-INFO +1 -1
- assemblyline_core-4.7.5.dev2/assemblyline_core/VERSION +1 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/updater/run_updater.py +24 -17
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core.egg-info/PKG-INFO +1 -1
- assemblyline_core-4.7.5.dev1/assemblyline_core/VERSION +0 -1
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/LICENCE.md +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/README.md +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/__init__.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/alerter/__init__.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/alerter/processing.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/alerter/run_alerter.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/archiver/__init__.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/archiver/run_archiver.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/badlist_client.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/dispatching/__init__.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/dispatching/client.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/dispatching/dispatcher.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/dispatching/schedules.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/expiry/__init__.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/expiry/run_expiry.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/ingester/__init__.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/ingester/constants.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/ingester/ingester.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/metrics/__init__.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/metrics/es_metrics.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/metrics/heartbeat_formatter.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/metrics/helper.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/metrics/metrics_server.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/metrics/run_heartbeat_manager.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/metrics/run_metrics_aggregator.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/metrics/run_statistics_aggregator.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/replay/__init__.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/replay/client.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/replay/creator/__init__.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/replay/creator/run.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/replay/creator/run_worker.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/replay/loader/__init__.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/replay/loader/run.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/replay/loader/run_worker.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/replay/replay.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/safelist_client.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/scaler/__init__.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/scaler/collection.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/scaler/controllers/__init__.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/scaler/controllers/docker_ctl.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/scaler/controllers/interface.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/scaler/controllers/kubernetes_ctl.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/scaler/run_scaler.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/scaler/scaler_server.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/server_base.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/signature_client.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/submission_client.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/tasking_client.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/updater/__init__.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/updater/helper.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/vacuum/__init__.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/vacuum/crawler.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/vacuum/department_map.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/vacuum/safelist.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/vacuum/stream_map.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/vacuum/worker.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/workflow/__init__.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/workflow/run_workflow.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core.egg-info/SOURCES.txt +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core.egg-info/dependency_links.txt +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core.egg-info/requires.txt +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core.egg-info/top_level.txt +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/setup.cfg +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/setup.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/test/test_alerter.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/test/test_badlist_client.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/test/test_expiry.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/test/test_replay.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/test/test_safelist_client.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/test/test_scaler.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/test/test_scheduler.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/test/test_signature_client.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/test/test_tasking_client.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/test/test_vacuum.py +0 -0
- {assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/test/test_workflow.py +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
4.7.5.dev2
|
|
@@ -489,12 +489,18 @@ class ServiceUpdater(ThreadedCoreBase):
|
|
|
489
489
|
# Add any mounts defined in the service defaults to the updater interface so they get applied to all updates/installs
|
|
490
490
|
self.mounts = self.config.core.scaler.service_defaults.mounts
|
|
491
491
|
|
|
492
|
+
# If there are no services installed push the auto install services to the install table
|
|
493
|
+
installed_service_names = list(self.datastore.service_delta.keys())
|
|
494
|
+
if len(installed_service_names) == 0:
|
|
495
|
+
for service in self.config.core.updater.auto_install:
|
|
496
|
+
self.container_install.add(service.name, {'image': service.image})
|
|
497
|
+
|
|
492
498
|
def _handle_service_change_event(self, data: Optional[ServiceChange]):
|
|
493
499
|
if data is not None:
|
|
494
500
|
if data.operation == Operation.Incompatible:
|
|
495
501
|
self.incompatible_services.add(data.name)
|
|
496
502
|
|
|
497
|
-
def container_installs(self):
|
|
503
|
+
def container_installs(self) -> None:
|
|
498
504
|
"""Go through the list of services and check what are the latest tags for it"""
|
|
499
505
|
while self.running:
|
|
500
506
|
self.log.info("[CI] Installing all services marked for install...")
|
|
@@ -561,8 +567,8 @@ class ServiceUpdater(ThreadedCoreBase):
|
|
|
561
567
|
)
|
|
562
568
|
|
|
563
569
|
except Exception as e:
|
|
564
|
-
self.log.error(
|
|
565
|
-
|
|
570
|
+
self.log.error("[CI] Service %s has failed to install. Install procedure cancelled... [%s]",
|
|
571
|
+
service_name, e)
|
|
566
572
|
finally:
|
|
567
573
|
self.container_keys.pop(api_key)
|
|
568
574
|
|
|
@@ -574,17 +580,18 @@ class ServiceUpdater(ThreadedCoreBase):
|
|
|
574
580
|
for service_name, install_data in self.container_install.items().items():
|
|
575
581
|
install_threads.append(service_installs_exec.submit(install_service, service_name, install_data))
|
|
576
582
|
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
583
|
+
# Check the status of the container installs
|
|
584
|
+
while install_threads:
|
|
585
|
+
# Delay each check by 5 seconds to give the container adequate time to perform version registration
|
|
586
|
+
time.sleep(5)
|
|
587
|
+
pending_threads = []
|
|
588
|
+
for thread in install_threads:
|
|
589
|
+
if not thread.done():
|
|
590
|
+
# Add thead to the list of still pending jobs
|
|
591
|
+
pending_threads.append(thread)
|
|
592
|
+
continue
|
|
593
|
+
|
|
594
|
+
service_key: str = thread.result()
|
|
588
595
|
service_name, latest_tag = service_key.split("_")
|
|
589
596
|
|
|
590
597
|
if self.datastore.service.get_if_exists(service_key):
|
|
@@ -612,13 +619,13 @@ class ServiceUpdater(ThreadedCoreBase):
|
|
|
612
619
|
f"service key ({service_key}) does not exist. Install procedure cancelled...")
|
|
613
620
|
self.container_install.pop(service_name)
|
|
614
621
|
|
|
615
|
-
|
|
616
|
-
|
|
622
|
+
# Update with still pending and loop until all are completed/failed
|
|
623
|
+
install_threads = pending_threads
|
|
617
624
|
|
|
618
625
|
# Clear out any old dead containers
|
|
619
626
|
self.controller.cleanup_stale()
|
|
620
627
|
|
|
621
|
-
self.log.info(
|
|
628
|
+
self.log.info("[CI] Done installing services, waiting {UPDATE_CHECK_INTERVAL} seconds for next install...")
|
|
622
629
|
time.sleep(UPDATE_CHECK_INTERVAL)
|
|
623
630
|
|
|
624
631
|
def container_updates(self):
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
4.7.5.dev1
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/alerter/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/archiver/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/badlist_client.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/expiry/__init__.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/expiry/run_expiry.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/ingester/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/ingester/ingester.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/metrics/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/metrics/helper.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/replay/__init__.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/replay/client.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/replay/loader/run.py
RENAMED
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/replay/replay.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/safelist_client.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/scaler/__init__.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/scaler/collection.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/scaler/run_scaler.py
RENAMED
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/server_base.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/signature_client.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/submission_client.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/tasking_client.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/updater/__init__.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/updater/helper.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/vacuum/__init__.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/vacuum/crawler.py
RENAMED
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/vacuum/safelist.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/vacuum/stream_map.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/vacuum/worker.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core/workflow/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.5.dev1 → assemblyline_core-4.7.5.dev2}/assemblyline_core.egg-info/SOURCES.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|