dtPyAppFramework 2.0__tar.gz → 2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dtpyappframework-2.0/src/dtPyAppFramework.egg-info → dtpyappframework-2.2}/PKG-INFO +1 -1
- dtpyappframework-2.2/src/dtPyAppFramework/_version.txt +1 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/application.py +10 -1
- {dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/process/__init__.py +9 -3
- {dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/process/multiprocessing.py +21 -6
- {dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/settings/__init__.py +6 -3
- {dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/settings/secrets/__init__.py +29 -43
- {dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/settings/secrets/keystore.py +1 -1
- {dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/settings/secrets/local_secret_store.py +2 -10
- dtpyappframework-2.2/src/dtPyAppFramework/settings/secrets/local_secret_stores_manager.py +284 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/settings/settings_reader.py +7 -4
- {dtpyappframework-2.0 → dtpyappframework-2.2/src/dtPyAppFramework.egg-info}/PKG-INFO +1 -1
- {dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework.egg-info/SOURCES.txt +1 -0
- dtpyappframework-2.0/src/dtPyAppFramework/_version.txt +0 -1
- {dtpyappframework-2.0 → dtpyappframework-2.2}/LICENCE.txt +0 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/MANIFEST.in +0 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/README.md +0 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/pyproject.toml +0 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/requirements.txt +0 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/setup.cfg +0 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/setup.py +0 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/__init__.py +0 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/_description.txt +0 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/_licence.txt +0 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/_metadata.yaml +0 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/_name.txt +0 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/cloud/__init__.py +0 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/cloud/aws.py +0 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/cloud/azure.py +0 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/cloud/cloud_session.py +0 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/decorators/__init__.py +0 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/logging/__init__.py +0 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/logging/default_logging.py +0 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/misc/__init__.py +0 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/misc/packaging/__init__.py +0 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/misc/yaml/__init__.py +0 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/paths/__init__.py +0 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/resources/__init__.py +0 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/settings/secrets/aws_secret_store.py +0 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/settings/secrets/azure_secret_store.py +0 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/settings/secrets/secret_store.py +0 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework.egg-info/dependency_links.txt +0 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework.egg-info/requires.txt +0 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework.egg-info/top_level.txt +0 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/tests/test_keystore.py +0 -0
- {dtpyappframework-2.0 → dtpyappframework-2.2}/tests/test_settings_reader.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: dtPyAppFramework
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.2
|
|
4
4
|
Summary: A Python library for common features in application development.
|
|
5
5
|
Author-email: Digital-Thought <dev@digital-thought.org>
|
|
6
6
|
Maintainer-email: Digital-Thought <dev@digital-thought.org>
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
2.2
|
|
@@ -89,6 +89,9 @@ class AbstractApp(object):
|
|
|
89
89
|
"""
|
|
90
90
|
raise NotImplementedError
|
|
91
91
|
|
|
92
|
+
def exiting(self):
|
|
93
|
+
raise NotImplementedError
|
|
94
|
+
|
|
92
95
|
def __define_args(self, arg_parser: ArgumentParser):
|
|
93
96
|
"""
|
|
94
97
|
Define command-line arguments based on application state.
|
|
@@ -116,6 +119,11 @@ class AbstractApp(object):
|
|
|
116
119
|
arg_parser.add_argument('--name', action='store', type=str, required=True, help="Secret Name")
|
|
117
120
|
arg_parser.add_argument('--value', action='store', type=str, required=True, help="Secret Value")
|
|
118
121
|
|
|
122
|
+
def exit(self):
|
|
123
|
+
self.exiting()
|
|
124
|
+
from dtPyAppFramework.settings import Settings
|
|
125
|
+
Settings().close()
|
|
126
|
+
|
|
119
127
|
def run(self):
|
|
120
128
|
"""
|
|
121
129
|
Run the application.
|
|
@@ -124,5 +132,6 @@ class AbstractApp(object):
|
|
|
124
132
|
self.__define_args(arg_parser)
|
|
125
133
|
self.process_manager = ProcessManager(description=self.description, version=self.version,
|
|
126
134
|
short_name=self.short_name, full_name=self.full_name,
|
|
127
|
-
console_app=self.console_app, main_procedure=self.main
|
|
135
|
+
console_app=self.console_app, main_procedure=self.main,
|
|
136
|
+
exit_procedure=self.exit)
|
|
128
137
|
self.process_manager.initialise_application(arg_parser)
|
|
@@ -45,7 +45,7 @@ class ProcessManager():
|
|
|
45
45
|
stderr_txt_file (file): File object for capturing stderr to a text file.
|
|
46
46
|
"""
|
|
47
47
|
|
|
48
|
-
def __init__(self, description, version, short_name, full_name, console_app, main_procedure):
|
|
48
|
+
def __init__(self, description, version, short_name, full_name, console_app, main_procedure, exit_procedure=None):
|
|
49
49
|
super().__init__()
|
|
50
50
|
# Initialization of attributes
|
|
51
51
|
self.description = description
|
|
@@ -59,11 +59,12 @@ class ProcessManager():
|
|
|
59
59
|
self.resource_manager = None
|
|
60
60
|
self.log_path = None
|
|
61
61
|
self.main_procedure = main_procedure
|
|
62
|
+
self.exit_procedure = exit_procedure
|
|
62
63
|
self.multiprocessing_manager = None
|
|
63
64
|
self.stdout_txt_file = None
|
|
64
65
|
self.stderr_txt_file = None
|
|
65
66
|
|
|
66
|
-
def __initialise_spawned_application__(self, parent_log_path, job_id, worker_id, job_name):
|
|
67
|
+
def __initialise_spawned_application__(self, parent_log_path, job_id, worker_id, job_name, pipe_registry):
|
|
67
68
|
"""
|
|
68
69
|
Initialize a spawned instance of the application in a multiprocessing environment.
|
|
69
70
|
|
|
@@ -84,7 +85,7 @@ class ProcessManager():
|
|
|
84
85
|
spawned_process=True,
|
|
85
86
|
job_id=job_id, worker_id=worker_id,
|
|
86
87
|
parent_log_path=parent_log_path)
|
|
87
|
-
self.application_settings.init_settings_readers()
|
|
88
|
+
self.application_settings.init_settings_readers(pipe_registry=pipe_registry)
|
|
88
89
|
self.application_settings.secret_manager.load_cloud_stores()
|
|
89
90
|
|
|
90
91
|
self.__initialise_stdout_capt__()
|
|
@@ -151,6 +152,8 @@ class ProcessManager():
|
|
|
151
152
|
|
|
152
153
|
except KeyboardInterrupt as kbi:
|
|
153
154
|
logging.warning('(KeyboardInterrupt) Exiting application.')
|
|
155
|
+
if self.exit_procedure:
|
|
156
|
+
self.exit_procedure()
|
|
154
157
|
if not self.console_app:
|
|
155
158
|
self.stdout_txt_file.close()
|
|
156
159
|
self.stderr_txt_file.close()
|
|
@@ -184,3 +187,6 @@ class ProcessManager():
|
|
|
184
187
|
"""
|
|
185
188
|
self.load_config()
|
|
186
189
|
self.main_procedure(args)
|
|
190
|
+
|
|
191
|
+
if self.exit_procedure:
|
|
192
|
+
self.exit_procedure()
|
{dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/process/multiprocessing.py
RENAMED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from ..decorators import singleton
|
|
2
2
|
from multiprocessing import Process, Manager
|
|
3
3
|
from ..paths import ApplicationPaths
|
|
4
|
+
from dtPyAppFramework.settings import Settings
|
|
4
5
|
|
|
5
6
|
import sys
|
|
6
7
|
import os
|
|
@@ -25,7 +26,7 @@ def get_new_uuid():
|
|
|
25
26
|
|
|
26
27
|
|
|
27
28
|
@singleton()
|
|
28
|
-
class MultiProcessingManager
|
|
29
|
+
class MultiProcessingManager:
|
|
29
30
|
"""
|
|
30
31
|
Singleton class for managing multiprocessing operations.
|
|
31
32
|
|
|
@@ -36,6 +37,7 @@ class MultiProcessingManager():
|
|
|
36
37
|
def __init__(self):
|
|
37
38
|
super().__init__()
|
|
38
39
|
self.log_path = None
|
|
40
|
+
self.jobs = {}
|
|
39
41
|
|
|
40
42
|
def set_log_path(self, log_path):
|
|
41
43
|
"""
|
|
@@ -46,6 +48,11 @@ class MultiProcessingManager():
|
|
|
46
48
|
"""
|
|
47
49
|
self.log_path = log_path
|
|
48
50
|
|
|
51
|
+
def get_job(self, job_name):
|
|
52
|
+
if job_name not in self.jobs:
|
|
53
|
+
return None
|
|
54
|
+
return self.jobs[job_name]
|
|
55
|
+
|
|
49
56
|
def new_multiprocessing_job(self, job_name, worker_count, target, args=(), kwargs={}):
|
|
50
57
|
"""
|
|
51
58
|
Create a new multiprocessing job.
|
|
@@ -60,7 +67,9 @@ class MultiProcessingManager():
|
|
|
60
67
|
Returns:
|
|
61
68
|
MultiProcessingJob: An instance of MultiProcessingJob.
|
|
62
69
|
"""
|
|
63
|
-
|
|
70
|
+
job = MultiProcessingJob(self.log_path, job_name, worker_count, target, args, kwargs)
|
|
71
|
+
self.jobs[job_name] = job
|
|
72
|
+
return job
|
|
64
73
|
|
|
65
74
|
|
|
66
75
|
class MultiProcessingJob():
|
|
@@ -94,8 +103,9 @@ class MultiProcessingJob():
|
|
|
94
103
|
Start the worker processes for the job.
|
|
95
104
|
"""
|
|
96
105
|
for x in range(self.worker_count):
|
|
97
|
-
|
|
98
|
-
|
|
106
|
+
pipe_registry = Settings().secret_manager.local_secrets_store_manager.server_thread.pipe_registry
|
|
107
|
+
worker = DtProcess(self.log_path, self.job_id, self.job_name, self.target, self.job_name, pipe_registry,
|
|
108
|
+
self.args, self.kwargs)
|
|
99
109
|
worker.start()
|
|
100
110
|
logging.info(f'Started Worker {worker.worker_id} for Job ID {self.job_id} ({self.job_name}).')
|
|
101
111
|
self.workers.append(worker)
|
|
@@ -109,6 +119,8 @@ class MultiProcessingJob():
|
|
|
109
119
|
logging.info(f'All Workers for Job ID {self.job_id} ({self.job_name}) ended.')
|
|
110
120
|
|
|
111
121
|
|
|
122
|
+
|
|
123
|
+
|
|
112
124
|
class DtProcess(Process):
|
|
113
125
|
"""
|
|
114
126
|
Custom multiprocessing Process class with additional attributes.
|
|
@@ -121,12 +133,14 @@ class DtProcess(Process):
|
|
|
121
133
|
multi_processing_job (MultiProcessingJob): Reference to the parent MultiProcessingJob.
|
|
122
134
|
"""
|
|
123
135
|
|
|
124
|
-
def __init__(self, parent_log_path, job_id, job_name, target=None, name=None,
|
|
136
|
+
def __init__(self, parent_log_path, job_id, job_name, target=None, name=None, pipe_registry=None,
|
|
137
|
+
args=(), kwargs={}):
|
|
125
138
|
self.worker_id = get_new_uuid()
|
|
126
139
|
self.job_id = job_id
|
|
127
140
|
self.job_name = job_name
|
|
128
141
|
self.parent_log_path = parent_log_path
|
|
129
142
|
self.multi_processing_job = None
|
|
143
|
+
self.pipe_registry = pipe_registry
|
|
130
144
|
super(DtProcess, self).__init__(target=target, name=name, args=args, kwargs=kwargs)
|
|
131
145
|
|
|
132
146
|
def set_parent(self, job):
|
|
@@ -144,8 +158,9 @@ class DtProcess(Process):
|
|
|
144
158
|
"""
|
|
145
159
|
from . import ProcessManager
|
|
146
160
|
ProcessManager().__initialise_spawned_application__(self.parent_log_path, self.job_id, self.worker_id,
|
|
147
|
-
self.job_name)
|
|
161
|
+
self.job_name, self.pipe_registry)
|
|
148
162
|
print(f'Process ID = {os.getpid()}')
|
|
149
163
|
print(f'worker_id {os.getpid()} = {self.worker_id}')
|
|
150
164
|
print(f'job_id {os.getpid()} = {self.job_id}')
|
|
151
165
|
super().run()
|
|
166
|
+
Settings().secret_manager.close()
|
|
@@ -36,12 +36,15 @@ class Settings(dict):
|
|
|
36
36
|
|
|
37
37
|
self.settings_readers = []
|
|
38
38
|
self.persistent_settings_stores = []
|
|
39
|
-
self.secret_manager = None
|
|
39
|
+
self.secret_manager: SecretsManager = None
|
|
40
40
|
self.cloud_session_manager = None
|
|
41
41
|
|
|
42
42
|
super().__init__()
|
|
43
43
|
|
|
44
|
-
def
|
|
44
|
+
def close(self):
|
|
45
|
+
self.secret_manager.close()
|
|
46
|
+
|
|
47
|
+
def init_settings_readers(self, pipe_registry=None):
|
|
45
48
|
"""
|
|
46
49
|
Initialize settings readers.
|
|
47
50
|
"""
|
|
@@ -53,7 +56,7 @@ class Settings(dict):
|
|
|
53
56
|
from ..cloud import CloudSessionManager
|
|
54
57
|
self.cloud_session_manager = CloudSessionManager()
|
|
55
58
|
self.secret_manager = SecretsManager(application_paths=self.application_paths, application_settings=self,
|
|
56
|
-
cloud_session_manager=self.cloud_session_manager)
|
|
59
|
+
cloud_session_manager=self.cloud_session_manager, pipe_registry=pipe_registry)
|
|
57
60
|
|
|
58
61
|
def persist_settings(self, settings, scope):
|
|
59
62
|
if scope == 'app':
|
{dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/settings/secrets/__init__.py
RENAMED
|
@@ -4,6 +4,7 @@ from ...decorators import singleton
|
|
|
4
4
|
from .local_secret_store import LocalSecretStore
|
|
5
5
|
from .aws_secret_store import AWSSecretsStore
|
|
6
6
|
from .azure_secret_store import AzureSecretsStore
|
|
7
|
+
from .local_secret_stores_manager import LocalSecretStoresManager
|
|
7
8
|
|
|
8
9
|
|
|
9
10
|
@singleton()
|
|
@@ -17,7 +18,7 @@ class SecretsManager(object):
|
|
|
17
18
|
stores (list): A list to hold different secret stores.
|
|
18
19
|
"""
|
|
19
20
|
|
|
20
|
-
def __init__(self, application_paths=None, application_settings=None, cloud_session_manager=None) -> None:
|
|
21
|
+
def __init__(self, application_paths=None, application_settings=None, cloud_session_manager=None, pipe_registry=None) -> None:
|
|
21
22
|
"""
|
|
22
23
|
Initialize the SecretsManager.
|
|
23
24
|
|
|
@@ -35,39 +36,16 @@ class SecretsManager(object):
|
|
|
35
36
|
if not self.application_paths:
|
|
36
37
|
self.application_paths = ApplicationPaths()
|
|
37
38
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
root_store_path=self.application_paths.usr_data_root_path,
|
|
42
|
-
application_settings=self.application_settings,
|
|
43
|
-
app_short_name=self.application_paths.app_short_name))
|
|
44
|
-
|
|
45
|
-
try:
|
|
46
|
-
# Add local app secret store (if it exists)
|
|
47
|
-
self.stores.append(LocalSecretStore(store_name="App_Local_Store",
|
|
48
|
-
store_priority=1,
|
|
49
|
-
root_store_path=self.application_paths.app_data_root_path,
|
|
50
|
-
application_settings=self.application_settings,
|
|
51
|
-
app_short_name=self.application_paths.app_short_name))
|
|
52
|
-
except Exception as ex:
|
|
53
|
-
print(f'Skipping APP Local Secret Store. {ex}')
|
|
39
|
+
self.local_secrets_store_manager = LocalSecretStoresManager(application_paths, application_settings,
|
|
40
|
+
pipe_registry)
|
|
41
|
+
self.store_names.extend(self.local_secrets_store_manager.store_names)
|
|
54
42
|
|
|
55
|
-
|
|
56
|
-
|
|
43
|
+
|
|
44
|
+
def close(self):
|
|
45
|
+
self.local_secrets_store_manager.close()
|
|
57
46
|
|
|
58
47
|
def get_local_stores_index(self):
|
|
59
|
-
|
|
60
|
-
for key in _index:
|
|
61
|
-
store: LocalSecretStore = self.get_store(key)
|
|
62
|
-
try:
|
|
63
|
-
_index[key]['available'] = store.store_available
|
|
64
|
-
_index[key]['index'] = store.get_index()
|
|
65
|
-
_index[key]['read_only'] = store.store_read_only
|
|
66
|
-
except Exception as ex:
|
|
67
|
-
logging.error(str(ex))
|
|
68
|
-
_index[key]['available'] = False
|
|
69
|
-
|
|
70
|
-
return _index
|
|
48
|
+
return self.local_secrets_store_manager.get_local_stores_index()
|
|
71
49
|
|
|
72
50
|
def _sort_stores(self):
|
|
73
51
|
self.stores.sort(key=lambda x: x.store_priority)
|
|
@@ -134,18 +112,22 @@ class SecretsManager(object):
|
|
|
134
112
|
store_name = elements[0]
|
|
135
113
|
key = elements[1]
|
|
136
114
|
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
115
|
+
if store_name is None or store_name in ['User_Local_Store', 'App_Local_Store']:
|
|
116
|
+
value = self.local_secrets_store_manager.get_secret(key, value, store_name)
|
|
117
|
+
|
|
118
|
+
if not value:
|
|
119
|
+
for store in self.stores:
|
|
120
|
+
if store_name and store_name == store.store_name:
|
|
121
|
+
if store.store_available:
|
|
122
|
+
value = store.get_secret(key, None)
|
|
123
|
+
else:
|
|
124
|
+
logging.error(f'Store {store.store_name} is not available to retrieve secret.')
|
|
125
|
+
break
|
|
126
|
+
elif not store_name:
|
|
127
|
+
if store.store_available:
|
|
128
|
+
value = store.get_secret(key, None)
|
|
129
|
+
if value:
|
|
130
|
+
break
|
|
149
131
|
|
|
150
132
|
if not value:
|
|
151
133
|
logging.debug(f'The Secret {key} was not found. Returning default value.')
|
|
@@ -173,6 +155,10 @@ class SecretsManager(object):
|
|
|
173
155
|
"""
|
|
174
156
|
if not store_name:
|
|
175
157
|
store_name = 'User_Local_Store'
|
|
158
|
+
|
|
159
|
+
if store_name in ['User_Local_Store', 'App_Local_Store']:
|
|
160
|
+
return self.local_secrets_store_manager.set_secret(key, value, store_name)
|
|
161
|
+
|
|
176
162
|
for store in self.stores:
|
|
177
163
|
if store_name == store.store_name:
|
|
178
164
|
if store.store_available and not store.store_read_only:
|
|
@@ -122,7 +122,6 @@ class LocalSecretStore(AbstractSecretStore):
|
|
|
122
122
|
self.delete_secret(key)
|
|
123
123
|
|
|
124
124
|
self.store.set(key=key, value=value)
|
|
125
|
-
self.__save()
|
|
126
125
|
|
|
127
126
|
|
|
128
127
|
def set_secret(self, key, value):
|
|
@@ -137,7 +136,6 @@ class LocalSecretStore(AbstractSecretStore):
|
|
|
137
136
|
self.delete_secret(key)
|
|
138
137
|
|
|
139
138
|
self.store.set(key=key, value=value)
|
|
140
|
-
self.__save()
|
|
141
139
|
index = self.get_index()
|
|
142
140
|
if key not in index:
|
|
143
141
|
index.append(key)
|
|
@@ -151,7 +149,6 @@ class LocalSecretStore(AbstractSecretStore):
|
|
|
151
149
|
key (str): Key of the secret.
|
|
152
150
|
"""
|
|
153
151
|
self.store.delete(key=key)
|
|
154
|
-
self.__save()
|
|
155
152
|
index = self.get_index()
|
|
156
153
|
while key in index:
|
|
157
154
|
index.remove(key)
|
|
@@ -160,7 +157,6 @@ class LocalSecretStore(AbstractSecretStore):
|
|
|
160
157
|
def __set_index(self, index: list):
|
|
161
158
|
logging.info(index)
|
|
162
159
|
self.store.set(key=f'{self.store_name}.INDEX', value=json.dumps(index))
|
|
163
|
-
self.__save()
|
|
164
160
|
|
|
165
161
|
def get_index(self) -> list:
|
|
166
162
|
index = self.get_secret(f'{self.store_name}.INDEX', None)
|
|
@@ -170,14 +166,10 @@ class LocalSecretStore(AbstractSecretStore):
|
|
|
170
166
|
|
|
171
167
|
return json.loads(index)
|
|
172
168
|
|
|
173
|
-
def __save(self):
|
|
174
|
-
"""Save the changes made to the local secret store."""
|
|
175
|
-
self.store.set('sstore_save', 'true')
|
|
176
|
-
self.store.delete('sstore_save')
|
|
177
|
-
|
|
178
169
|
def __is_writeable(self):
|
|
179
170
|
try:
|
|
180
|
-
self.
|
|
171
|
+
self.store.set('sstore_save', 'true')
|
|
172
|
+
self.store.delete('sstore_save')
|
|
181
173
|
return True
|
|
182
174
|
except Exception as ex:
|
|
183
175
|
logging.warning(str(ex))
|
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
from ...paths import ApplicationPaths
|
|
2
|
+
from .local_secret_store import LocalSecretStore
|
|
3
|
+
import multiprocessing
|
|
4
|
+
from threading import Thread
|
|
5
|
+
import logging
|
|
6
|
+
from enum import Enum
|
|
7
|
+
|
|
8
|
+
class LocalSecretsManagerCommands(Enum):
|
|
9
|
+
CMD_GET_SECRET = 1
|
|
10
|
+
CMD_SET_SECRET = 2
|
|
11
|
+
CMD_SET_PERSISTENT_SECRET = 3
|
|
12
|
+
CMD_DELETE_SECRET = 4
|
|
13
|
+
CMD_STORE_AVAILABLE = 5
|
|
14
|
+
CMD_GET_INDEX = 6
|
|
15
|
+
CMD_STORE_READ_ONLY = 7
|
|
16
|
+
CMD_EXIT = 8
|
|
17
|
+
CMD_SHUTDOWN = 9
|
|
18
|
+
CMD_SUCCESS = 10
|
|
19
|
+
CMD_STORE_NAMES = 11
|
|
20
|
+
|
|
21
|
+
class LocalSecretsManagerServer(Thread):
|
|
22
|
+
|
|
23
|
+
def __init__(self, application_paths, application_settings):
|
|
24
|
+
super().__init__(name='LocalSecretStoresManager_ServerThread')
|
|
25
|
+
self.application_paths = application_paths
|
|
26
|
+
self.application_settings = application_settings
|
|
27
|
+
self.stores = []
|
|
28
|
+
self.store_names = []
|
|
29
|
+
self.pipe_registry = multiprocessing.Queue()
|
|
30
|
+
self._load_stores()
|
|
31
|
+
|
|
32
|
+
def _load_stores(self):
|
|
33
|
+
if not self.application_paths:
|
|
34
|
+
self.application_paths = ApplicationPaths()
|
|
35
|
+
|
|
36
|
+
self.stores.append(LocalSecretStore(store_name="User_Local_Store",
|
|
37
|
+
store_priority=-0,
|
|
38
|
+
root_store_path=self.application_paths.usr_data_root_path,
|
|
39
|
+
application_settings=self.application_settings,
|
|
40
|
+
app_short_name=self.application_paths.app_short_name))
|
|
41
|
+
|
|
42
|
+
try:
|
|
43
|
+
# Add local app secret store (if it exists)
|
|
44
|
+
self.stores.append(LocalSecretStore(store_name="App_Local_Store",
|
|
45
|
+
store_priority=1,
|
|
46
|
+
root_store_path=self.application_paths.app_data_root_path,
|
|
47
|
+
application_settings=self.application_settings,
|
|
48
|
+
app_short_name=self.application_paths.app_short_name))
|
|
49
|
+
except Exception as ex:
|
|
50
|
+
print(f'Skipping APP Local Secret Store. {ex}')
|
|
51
|
+
|
|
52
|
+
for store in self.stores:
|
|
53
|
+
self.store_names.append(store.store_name)
|
|
54
|
+
|
|
55
|
+
def run(self):
|
|
56
|
+
running = True
|
|
57
|
+
while running:
|
|
58
|
+
conn = self.pipe_registry.get() # Iterate over a copy to allow dynamic modifications
|
|
59
|
+
if conn.poll(): # Check if there is a message from this pipe
|
|
60
|
+
try:
|
|
61
|
+
command, key, value, store_name = conn.recv()
|
|
62
|
+
match command:
|
|
63
|
+
case LocalSecretsManagerCommands.CMD_GET_SECRET:
|
|
64
|
+
conn.send(self._get_secret(key, value, store_name))
|
|
65
|
+
|
|
66
|
+
case LocalSecretsManagerCommands.CMD_SET_SECRET:
|
|
67
|
+
conn.send(self._set_secret(key, value, store_name))
|
|
68
|
+
|
|
69
|
+
case LocalSecretsManagerCommands.CMD_SET_PERSISTENT_SECRET:
|
|
70
|
+
conn.send(self._set_persistent_setting(key, value))
|
|
71
|
+
|
|
72
|
+
case LocalSecretsManagerCommands.CMD_DELETE_SECRET:
|
|
73
|
+
conn.send(self._delete_secret(key, store_name))
|
|
74
|
+
|
|
75
|
+
case LocalSecretsManagerCommands.CMD_STORE_AVAILABLE:
|
|
76
|
+
conn.send(self._store_available(store_name))
|
|
77
|
+
|
|
78
|
+
case LocalSecretsManagerCommands.CMD_GET_INDEX:
|
|
79
|
+
conn.send(self._get_index(store_name))
|
|
80
|
+
|
|
81
|
+
case LocalSecretsManagerCommands.CMD_STORE_READ_ONLY:
|
|
82
|
+
conn.send(self._store_read_only(store_name))
|
|
83
|
+
|
|
84
|
+
case LocalSecretsManagerCommands.CMD_STORE_NAMES:
|
|
85
|
+
conn.send(self.store_names)
|
|
86
|
+
|
|
87
|
+
case LocalSecretsManagerCommands.CMD_EXIT:
|
|
88
|
+
conn.close()
|
|
89
|
+
logging.warning('Closed Connection following EXIT request.')
|
|
90
|
+
|
|
91
|
+
case LocalSecretsManagerCommands.CMD_SHUTDOWN:
|
|
92
|
+
running = False
|
|
93
|
+
|
|
94
|
+
case _:
|
|
95
|
+
logging.error(
|
|
96
|
+
f'Unrecognised Request: command = {command}, key = {key}, value = {value}, store_name = {store_name}')
|
|
97
|
+
|
|
98
|
+
if not conn.closed:
|
|
99
|
+
self.pipe_registry.put(conn)
|
|
100
|
+
|
|
101
|
+
except EOFError as ex:
|
|
102
|
+
logging.exception(ex)
|
|
103
|
+
|
|
104
|
+
else:
|
|
105
|
+
self.pipe_registry.put(conn)
|
|
106
|
+
|
|
107
|
+
while not self.pipe_registry.empty():
|
|
108
|
+
pipe = self.pipe_registry.get()
|
|
109
|
+
pipe.close()
|
|
110
|
+
logging.warning('Closed All Connections following SHUTDOWN request.')
|
|
111
|
+
return
|
|
112
|
+
|
|
113
|
+
def _get_store(self, store_name):
|
|
114
|
+
"""
|
|
115
|
+
Get a secret store by name.
|
|
116
|
+
|
|
117
|
+
Args:
|
|
118
|
+
store_name (str): Name of the secret store.
|
|
119
|
+
|
|
120
|
+
Returns:
|
|
121
|
+
Secret store instance if found, else None.
|
|
122
|
+
"""
|
|
123
|
+
for store in self.stores:
|
|
124
|
+
if store_name == store.store_name:
|
|
125
|
+
return store
|
|
126
|
+
return None
|
|
127
|
+
|
|
128
|
+
def _store_available(self, store_name):
|
|
129
|
+
store: LocalSecretStore = self._get_store(store_name)
|
|
130
|
+
return store.store_available
|
|
131
|
+
|
|
132
|
+
def _get_index(self, store_name):
|
|
133
|
+
store: LocalSecretStore = self._get_store(store_name)
|
|
134
|
+
return store.get_index()
|
|
135
|
+
|
|
136
|
+
def _store_read_only(self, store_name):
|
|
137
|
+
store: LocalSecretStore = self._get_store(store_name)
|
|
138
|
+
return store.store_read_only
|
|
139
|
+
|
|
140
|
+
def _get_secret(self, key, default_value=None, store_name=None):
|
|
141
|
+
value = None
|
|
142
|
+
if len(key.split(".")) == 2 and key.split(".")[0] in self.store_names:
|
|
143
|
+
elements = key.split(".")
|
|
144
|
+
store_name = elements[0]
|
|
145
|
+
key = elements[1]
|
|
146
|
+
|
|
147
|
+
for store in self.stores:
|
|
148
|
+
if store_name and store_name == store.store_name:
|
|
149
|
+
if store.store_available:
|
|
150
|
+
value = store.get_secret(key, None)
|
|
151
|
+
else:
|
|
152
|
+
logging.error(f'Store {store.store_name} is not available to retrieve secret.')
|
|
153
|
+
break
|
|
154
|
+
elif not store_name:
|
|
155
|
+
if store.store_available:
|
|
156
|
+
value = store.get_secret(key, None)
|
|
157
|
+
if value:
|
|
158
|
+
break
|
|
159
|
+
|
|
160
|
+
if not value:
|
|
161
|
+
logging.debug(f'The Secret {key} was not found. Returning default value.')
|
|
162
|
+
value = default_value
|
|
163
|
+
|
|
164
|
+
return value
|
|
165
|
+
|
|
166
|
+
def _set_persistent_setting(self, key, value):
|
|
167
|
+
for store in self.stores:
|
|
168
|
+
if 'User_Local_Store' == store.store_name:
|
|
169
|
+
if store.store_available and not store.store_read_only:
|
|
170
|
+
store.set_persistent_setting(key, value)
|
|
171
|
+
else:
|
|
172
|
+
logging.warning(f'Secrets Store {store.store_name} is either not available or is read only.')
|
|
173
|
+
break
|
|
174
|
+
return LocalSecretsManagerCommands.CMD_SUCCESS
|
|
175
|
+
|
|
176
|
+
def _set_secret(self, key, value, store_name='User_Local_Store'):
|
|
177
|
+
if not store_name:
|
|
178
|
+
store_name = 'User_Local_Store'
|
|
179
|
+
for store in self.stores:
|
|
180
|
+
if store_name == store.store_name:
|
|
181
|
+
if store.store_available and not store.store_read_only:
|
|
182
|
+
store.set_secret(key, value)
|
|
183
|
+
else:
|
|
184
|
+
logging.warning(f'Secrets Store {store.store_name} is either not available or is read only.')
|
|
185
|
+
break
|
|
186
|
+
return LocalSecretsManagerCommands.CMD_SUCCESS
|
|
187
|
+
|
|
188
|
+
def _delete_secret(self, key, store_name='User_Local_Store'):
|
|
189
|
+
for store in self.stores:
|
|
190
|
+
if store_name and store_name == store.name():
|
|
191
|
+
value = store.delete_secret(key)
|
|
192
|
+
break
|
|
193
|
+
|
|
194
|
+
return LocalSecretsManagerCommands.CMD_SUCCESS
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
class LocalSecretStoresManager:
|
|
198
|
+
|
|
199
|
+
def __init__(self, application_paths=None, application_settings=None, pipe_registry=None):
|
|
200
|
+
self.application_paths = application_paths
|
|
201
|
+
self.application_settings = application_settings
|
|
202
|
+
self.child_connection = None
|
|
203
|
+
self.server_thread = None
|
|
204
|
+
self.pipe_registry = pipe_registry
|
|
205
|
+
|
|
206
|
+
if self.pipe_registry is None and self.server_thread is None:
|
|
207
|
+
self.server_thread = LocalSecretsManagerServer(self.application_paths, self.application_settings)
|
|
208
|
+
self.server_thread.start()
|
|
209
|
+
|
|
210
|
+
control_pipe_parent, self.child_connection = multiprocessing.Pipe()
|
|
211
|
+
self.server_thread.pipe_registry.put(control_pipe_parent)
|
|
212
|
+
|
|
213
|
+
elif self.pipe_registry is not None:
|
|
214
|
+
logging.info('t')
|
|
215
|
+
parent_conn, self.child_connection = multiprocessing.Pipe()
|
|
216
|
+
logging.info('3')
|
|
217
|
+
self.pipe_registry = pipe_registry
|
|
218
|
+
self.pipe_registry.put(parent_conn)
|
|
219
|
+
logging.info('4')
|
|
220
|
+
|
|
221
|
+
else:
|
|
222
|
+
logging.error('Unexpected State for LocalSecretStoresManager')
|
|
223
|
+
raise Exception('Unexpected State for LocalSecretStoresManager')
|
|
224
|
+
|
|
225
|
+
def close(self):
|
|
226
|
+
if self.pipe_registry is None and self.server_thread is not None:
|
|
227
|
+
self.child_connection.send((LocalSecretsManagerCommands.CMD_SHUTDOWN, None, None, None))
|
|
228
|
+
elif self.pipe_registry is not None:
|
|
229
|
+
self.child_connection.send((LocalSecretsManagerCommands.CMD_EXIT, None, None, None))
|
|
230
|
+
else:
|
|
231
|
+
logging.error('Unexpected State for LocalSecretStoresManager')
|
|
232
|
+
raise Exception('Unexpected State for LocalSecretStoresManager')
|
|
233
|
+
|
|
234
|
+
def get_local_stores_index(self):
|
|
235
|
+
_index = {"User_Local_Store": {}, "App_Local_Store": {}}
|
|
236
|
+
for key in _index:
|
|
237
|
+
try:
|
|
238
|
+
_index[key]['available'] = self.store_available(key)
|
|
239
|
+
_index[key]['index'] = self.get_index(key)
|
|
240
|
+
_index[key]['read_only'] = self.store_read_only(key)
|
|
241
|
+
except Exception as ex:
|
|
242
|
+
logging.error(str(ex))
|
|
243
|
+
_index[key]['available'] = False
|
|
244
|
+
|
|
245
|
+
return _index
|
|
246
|
+
|
|
247
|
+
@property
|
|
248
|
+
def store_names(self):
|
|
249
|
+
self.child_connection.send((LocalSecretsManagerCommands.CMD_STORE_NAMES, None, None, None))
|
|
250
|
+
return self.child_connection.recv()
|
|
251
|
+
|
|
252
|
+
def store_available(self, store_name):
|
|
253
|
+
self.child_connection.send((LocalSecretsManagerCommands.CMD_STORE_AVAILABLE, None, None, store_name))
|
|
254
|
+
return self.child_connection.recv()
|
|
255
|
+
|
|
256
|
+
def get_index(self, store_name):
|
|
257
|
+
self.child_connection.send((LocalSecretsManagerCommands.CMD_GET_INDEX, None, None, store_name))
|
|
258
|
+
return self.child_connection.recv()
|
|
259
|
+
|
|
260
|
+
def store_read_only(self, store_name):
|
|
261
|
+
self.child_connection.send((LocalSecretsManagerCommands.CMD_STORE_READ_ONLY, None, None, store_name))
|
|
262
|
+
return self.child_connection.recv()
|
|
263
|
+
|
|
264
|
+
def get_secret(self, key, default_value=None, store_name=None):
|
|
265
|
+
self.child_connection.send((LocalSecretsManagerCommands.CMD_GET_SECRET, key, default_value, store_name))
|
|
266
|
+
return self.child_connection.recv()
|
|
267
|
+
|
|
268
|
+
def set_persistent_setting(self, key, value):
|
|
269
|
+
self.child_connection.send((LocalSecretsManagerCommands.CMD_SET_PERSISTENT_SECRET, key, value, None))
|
|
270
|
+
resp = self.child_connection.recv()
|
|
271
|
+
if resp != LocalSecretsManagerCommands.CMD_SUCCESS:
|
|
272
|
+
logging.error(resp)
|
|
273
|
+
|
|
274
|
+
def set_secret(self, key, value, store_name='User_Local_Store'):
|
|
275
|
+
self.child_connection.send((LocalSecretsManagerCommands.CMD_SET_SECRET, key, value, store_name))
|
|
276
|
+
resp = self.child_connection.recv()
|
|
277
|
+
if resp != LocalSecretsManagerCommands.CMD_SUCCESS:
|
|
278
|
+
logging.error(resp)
|
|
279
|
+
|
|
280
|
+
def delete_secret(self, key, store_name='User_Local_Store'):
|
|
281
|
+
self.child_connection.send((LocalSecretsManagerCommands.CMD_DELETE_SECRET, key, store_name))
|
|
282
|
+
resp = self.child_connection.recv()
|
|
283
|
+
if resp != LocalSecretsManagerCommands.CMD_SUCCESS:
|
|
284
|
+
logging.error(resp)
|
{dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/settings/settings_reader.py
RENAMED
|
@@ -39,9 +39,11 @@ class ConfigFileWatcher(FileSystemEventHandler):
|
|
|
39
39
|
logging.warning(f'Config Watch File Deleted: {event.src_path}')
|
|
40
40
|
self.delete_action()
|
|
41
41
|
elif event.event_type == 'modified':
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
42
|
+
new_sha256 = self.calculate_sha256(event.src_path)
|
|
43
|
+
if new_sha256 != self.watch_file_sha256:
|
|
44
|
+
logging.warning(f'Config Watch File Changed: {event.src_path} - WAS: {self.watch_file_sha256},'
|
|
45
|
+
f' NOW: {new_sha256}')
|
|
46
|
+
self.watch_file_sha256 = new_sha256
|
|
45
47
|
self.change_action()
|
|
46
48
|
elif event.event_type == 'created':
|
|
47
49
|
logging.warning(f'Config Watch File Created: {event.src_path}')
|
|
@@ -88,7 +90,8 @@ class SettingsReader(dict):
|
|
|
88
90
|
event_handler = ConfigFileWatcher(change_action=self.load_yaml_file, delete_action=super().clear,
|
|
89
91
|
watch_file=self.CONFIG_FILE, watch_folder=path)
|
|
90
92
|
self.observer.schedule(event_handler, path, recursive=False)
|
|
91
|
-
|
|
93
|
+
if os.path.exists(path):
|
|
94
|
+
self.observer.start()
|
|
92
95
|
|
|
93
96
|
super().__init__()
|
|
94
97
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: dtPyAppFramework
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.2
|
|
4
4
|
Summary: A Python library for common features in application development.
|
|
5
5
|
Author-email: Digital-Thought <dev@digital-thought.org>
|
|
6
6
|
Maintainer-email: Digital-Thought <dev@digital-thought.org>
|
|
@@ -37,6 +37,7 @@ src/dtPyAppFramework/settings/secrets/aws_secret_store.py
|
|
|
37
37
|
src/dtPyAppFramework/settings/secrets/azure_secret_store.py
|
|
38
38
|
src/dtPyAppFramework/settings/secrets/keystore.py
|
|
39
39
|
src/dtPyAppFramework/settings/secrets/local_secret_store.py
|
|
40
|
+
src/dtPyAppFramework/settings/secrets/local_secret_stores_manager.py
|
|
40
41
|
src/dtPyAppFramework/settings/secrets/secret_store.py
|
|
41
42
|
tests/test_keystore.py
|
|
42
43
|
tests/test_settings_reader.py
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
2.0
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/logging/default_logging.py
RENAMED
|
File without changes
|
|
File without changes
|
{dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/misc/packaging/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework/settings/secrets/secret_store.py
RENAMED
|
File without changes
|
{dtpyappframework-2.0 → dtpyappframework-2.2}/src/dtPyAppFramework.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|