deltafi 2.14.0__py3-none-any.whl → 2.15.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of deltafi might be problematic. Click here for more details.
deltafi/action.py
CHANGED
|
@@ -38,10 +38,184 @@ class Join(ABC):
|
|
|
38
38
|
return TransformInput(content=all_content, metadata=all_metadata)
|
|
39
39
|
|
|
40
40
|
|
|
41
|
+
class ContentSpec:
|
|
42
|
+
name: str
|
|
43
|
+
media_type: str
|
|
44
|
+
description: str
|
|
45
|
+
|
|
46
|
+
def __init__(self, name: str = None, media_type: str = None, description: str = None):
|
|
47
|
+
self.name = name
|
|
48
|
+
self.media_type = media_type
|
|
49
|
+
self.description = description
|
|
50
|
+
|
|
51
|
+
def json(self):
|
|
52
|
+
json_dictionary = {}
|
|
53
|
+
if self.name is not None:
|
|
54
|
+
json_dictionary['name'] = self.name
|
|
55
|
+
if self.media_type is not None:
|
|
56
|
+
json_dictionary['mediaType'] = self.media_type
|
|
57
|
+
if self.description is not None:
|
|
58
|
+
json_dictionary['description'] = self.description
|
|
59
|
+
return json_dictionary
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class KeyedDescription:
|
|
63
|
+
key: str
|
|
64
|
+
description: str
|
|
65
|
+
|
|
66
|
+
def __init__(self, key: str, description: str):
|
|
67
|
+
self.key = key
|
|
68
|
+
self.description = description
|
|
69
|
+
|
|
70
|
+
def json(self):
|
|
71
|
+
json_dictionary = {}
|
|
72
|
+
if self.key is not None:
|
|
73
|
+
json_dictionary['key'] = self.key
|
|
74
|
+
json_dictionary['description'] = self.description
|
|
75
|
+
return json_dictionary
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class InputSpec:
|
|
79
|
+
content_summary: str
|
|
80
|
+
content_specs: List[ContentSpec]
|
|
81
|
+
metadata_summary: str
|
|
82
|
+
metadata_descriptions: List[KeyedDescription]
|
|
83
|
+
|
|
84
|
+
def __init__(self, content_summary: str = None, content_specs: List[ContentSpec] = None,
|
|
85
|
+
metadata_summary: str = None, metadata_descriptions: List[KeyedDescription] = None):
|
|
86
|
+
self.content_summary = content_summary
|
|
87
|
+
self.content_specs = content_specs
|
|
88
|
+
self.metadata_summary = metadata_summary
|
|
89
|
+
self.metadata_descriptions = metadata_descriptions
|
|
90
|
+
|
|
91
|
+
def json(self):
|
|
92
|
+
json_dictionary = {}
|
|
93
|
+
if self.content_summary is not None:
|
|
94
|
+
json_dictionary['contentSummary'] = self.content_summary
|
|
95
|
+
if self.content_specs is not None:
|
|
96
|
+
json_dictionary['contentSpecs'] = [cs.json() for cs in self.content_specs]
|
|
97
|
+
if self.metadata_summary is not None:
|
|
98
|
+
json_dictionary['metadataSummary'] = self.metadata_summary
|
|
99
|
+
if self.metadata_descriptions is not None:
|
|
100
|
+
json_dictionary['metadataDescriptions'] = [md.json() for md in self.metadata_descriptions]
|
|
101
|
+
return json_dictionary
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class OutputSpec:
|
|
105
|
+
content_summary: str
|
|
106
|
+
content_specs: List[ContentSpec]
|
|
107
|
+
metadata_summary: str
|
|
108
|
+
metadata_descriptions: List[KeyedDescription]
|
|
109
|
+
passthrough: bool
|
|
110
|
+
annotations_summary: str
|
|
111
|
+
annotation_descriptions: List[KeyedDescription]
|
|
112
|
+
|
|
113
|
+
def __init__(self, content_summary: str = None, content_specs: List[ContentSpec] = None,
|
|
114
|
+
metadata_summary: str = None, metadata_descriptions: List[KeyedDescription] = None,
|
|
115
|
+
passthrough: bool = False, annotations_summary: str = None,
|
|
116
|
+
annotation_descriptions: List[KeyedDescription] = None):
|
|
117
|
+
self.content_summary = content_summary
|
|
118
|
+
self.content_specs = content_specs
|
|
119
|
+
self.metadata_summary = metadata_summary
|
|
120
|
+
self.metadata_descriptions = metadata_descriptions
|
|
121
|
+
self.passthrough = passthrough
|
|
122
|
+
self.annotations_summary = annotations_summary
|
|
123
|
+
self.annotation_descriptions = annotation_descriptions
|
|
124
|
+
|
|
125
|
+
def json(self):
|
|
126
|
+
json_dictionary = {}
|
|
127
|
+
if self.content_summary is not None:
|
|
128
|
+
json_dictionary['contentSummary'] = self.content_summary
|
|
129
|
+
if self.content_specs is not None:
|
|
130
|
+
json_dictionary['contentSpecs'] = [cs.json() for cs in self.content_specs]
|
|
131
|
+
if self.metadata_summary is not None:
|
|
132
|
+
json_dictionary['metadataSummary'] = self.metadata_summary
|
|
133
|
+
if self.metadata_descriptions is not None:
|
|
134
|
+
json_dictionary['metadataDescriptions'] = [md.json() for md in self.metadata_descriptions]
|
|
135
|
+
if self.passthrough is not None:
|
|
136
|
+
json_dictionary['passthrough'] = self.passthrough
|
|
137
|
+
if self.annotations_summary is not None:
|
|
138
|
+
json_dictionary['annotationsSummary'] = self.annotations_summary
|
|
139
|
+
if self.annotation_descriptions is not None:
|
|
140
|
+
json_dictionary['annotationDescriptions'] = [ad.json() for ad in self.annotation_descriptions]
|
|
141
|
+
return json_dictionary
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
class DescriptionWithConditions:
|
|
145
|
+
description: str
|
|
146
|
+
conditions: List[str]
|
|
147
|
+
|
|
148
|
+
def __init__(self, description: str = None, conditions: List[str] = None):
|
|
149
|
+
self.description = description
|
|
150
|
+
self.conditions = conditions
|
|
151
|
+
|
|
152
|
+
def json(self):
|
|
153
|
+
json_dictionary = {}
|
|
154
|
+
if self.description is not None:
|
|
155
|
+
json_dictionary['description'] = self.description
|
|
156
|
+
if self.conditions is not None:
|
|
157
|
+
json_dictionary['conditions'] = [c for c in self.conditions]
|
|
158
|
+
return json_dictionary
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
class ActionOptions:
|
|
162
|
+
description: str
|
|
163
|
+
input_spec: InputSpec
|
|
164
|
+
output_spec: OutputSpec
|
|
165
|
+
filters: List[DescriptionWithConditions] = None
|
|
166
|
+
errors: List[DescriptionWithConditions] = None
|
|
167
|
+
notes: List[str]
|
|
168
|
+
details: str
|
|
169
|
+
|
|
170
|
+
def __init__(self, description: str = None, input_spec: InputSpec = None, output_spec: OutputSpec = None,
|
|
171
|
+
filters: List = None, errors: List = None, notes: List[str] = None, details: str = None):
|
|
172
|
+
self.description = description
|
|
173
|
+
self.input_spec = input_spec
|
|
174
|
+
self.output_spec = output_spec
|
|
175
|
+
if filters is not None:
|
|
176
|
+
self.filters = []
|
|
177
|
+
for f in filters:
|
|
178
|
+
if isinstance(f, DescriptionWithConditions):
|
|
179
|
+
self.filters.append(f)
|
|
180
|
+
else:
|
|
181
|
+
self.filters.append(DescriptionWithConditions(description=f))
|
|
182
|
+
if errors is not None:
|
|
183
|
+
self.errors = []
|
|
184
|
+
for e in errors:
|
|
185
|
+
if isinstance(e, DescriptionWithConditions):
|
|
186
|
+
self.errors.append(e)
|
|
187
|
+
else:
|
|
188
|
+
self.errors.append(DescriptionWithConditions(description=e))
|
|
189
|
+
self.notes = notes
|
|
190
|
+
self.details = details
|
|
191
|
+
|
|
192
|
+
def json(self):
|
|
193
|
+
json_dictionary = {}
|
|
194
|
+
if self.description is not None:
|
|
195
|
+
json_dictionary['description'] = self.description
|
|
196
|
+
if self.input_spec is not None:
|
|
197
|
+
json_dictionary['inputSpec'] = self.input_spec.json()
|
|
198
|
+
if self.output_spec is not None:
|
|
199
|
+
json_dictionary['outputSpec'] = self.output_spec.json()
|
|
200
|
+
if self.filters is not None:
|
|
201
|
+
json_dictionary['filters'] = [f.json() for f in self.filters]
|
|
202
|
+
if self.errors is not None:
|
|
203
|
+
json_dictionary['errors'] = [e.json() for e in self.errors]
|
|
204
|
+
if self.notes is not None:
|
|
205
|
+
json_dictionary['notes'] = [n for n in self.notes]
|
|
206
|
+
if self.details is not None:
|
|
207
|
+
json_dictionary['details'] = self.details
|
|
208
|
+
return json_dictionary
|
|
209
|
+
|
|
210
|
+
|
|
41
211
|
class Action(ABC):
|
|
42
|
-
def __init__(self, action_type: ActionType, description: str, valid_result_types: tuple
|
|
212
|
+
def __init__(self, action_type: ActionType, description: str, valid_result_types: tuple,
|
|
213
|
+
action_options: ActionOptions = None):
|
|
43
214
|
self.action_type = action_type
|
|
44
|
-
|
|
215
|
+
if action_options is None:
|
|
216
|
+
self.action_options = ActionOptions(description=description)
|
|
217
|
+
else:
|
|
218
|
+
self.action_options = action_options
|
|
45
219
|
self.valid_result_types = valid_result_types
|
|
46
220
|
|
|
47
221
|
@abstractmethod
|
|
@@ -92,8 +266,8 @@ class Action(ABC):
|
|
|
92
266
|
|
|
93
267
|
|
|
94
268
|
class EgressAction(Action, ABC):
|
|
95
|
-
def __init__(self, description: str):
|
|
96
|
-
super().__init__(ActionType.EGRESS, description, (EgressResult, ErrorResult, FilterResult))
|
|
269
|
+
def __init__(self, description: str, action_options: ActionOptions = None):
|
|
270
|
+
super().__init__(ActionType.EGRESS, description, (EgressResult, ErrorResult, FilterResult), action_options)
|
|
97
271
|
|
|
98
272
|
def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
|
|
99
273
|
return EgressInput(content=delta_file_message.content_list[0], metadata=delta_file_message.metadata)
|
|
@@ -107,8 +281,8 @@ class EgressAction(Action, ABC):
|
|
|
107
281
|
|
|
108
282
|
|
|
109
283
|
class TimedIngressAction(Action, ABC):
|
|
110
|
-
def __init__(self, description: str):
|
|
111
|
-
super().__init__(ActionType.TIMED_INGRESS, description, (IngressResult, ErrorResult))
|
|
284
|
+
def __init__(self, description: str, action_options: ActionOptions = None):
|
|
285
|
+
super().__init__(ActionType.TIMED_INGRESS, description, (IngressResult, ErrorResult), action_options)
|
|
112
286
|
|
|
113
287
|
def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
|
|
114
288
|
return None
|
|
@@ -122,9 +296,9 @@ class TimedIngressAction(Action, ABC):
|
|
|
122
296
|
|
|
123
297
|
|
|
124
298
|
class TransformAction(Action, ABC):
|
|
125
|
-
def __init__(self, description: str):
|
|
299
|
+
def __init__(self, description: str, action_options: ActionOptions = None):
|
|
126
300
|
super().__init__(ActionType.TRANSFORM, description,
|
|
127
|
-
(TransformResult, TransformResults, ErrorResult, FilterResult))
|
|
301
|
+
(TransformResult, TransformResults, ErrorResult, FilterResult), action_options)
|
|
128
302
|
|
|
129
303
|
def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
|
|
130
304
|
return TransformInput(content=delta_file_message.content_list, metadata=delta_file_message.metadata)
|
deltafi/plugin.py
CHANGED
|
@@ -29,7 +29,7 @@ from datetime import datetime, timezone, timedelta
|
|
|
29
29
|
from importlib import metadata
|
|
30
30
|
from os.path import isdir, isfile, join
|
|
31
31
|
from pathlib import Path
|
|
32
|
-
from typing import List
|
|
32
|
+
from typing import List, NamedTuple
|
|
33
33
|
|
|
34
34
|
import requests
|
|
35
35
|
import yaml
|
|
@@ -250,25 +250,13 @@ class Plugin(object):
|
|
|
250
250
|
def action_name(self, action):
|
|
251
251
|
return f"{self.coordinates.group_id}.{action.__class__.__name__}"
|
|
252
252
|
|
|
253
|
-
def _load_action_docs(self, action):
|
|
254
|
-
docs_path = str(Path(os.path.dirname(os.path.abspath(sys.argv[0]))) / 'docs')
|
|
255
|
-
if not isdir(docs_path):
|
|
256
|
-
return None
|
|
257
|
-
|
|
258
|
-
action_docs_file = join(docs_path, action.__class__.__name__ + '.md')
|
|
259
|
-
if not isfile(action_docs_file):
|
|
260
|
-
return None
|
|
261
|
-
|
|
262
|
-
return open(action_docs_file).read()
|
|
263
|
-
|
|
264
253
|
def _action_json(self, action):
|
|
265
254
|
return {
|
|
266
255
|
'name': self.action_name(action),
|
|
267
|
-
'description': action.description,
|
|
268
256
|
'type': action.action_type.name,
|
|
269
257
|
'supportsJoin': isinstance(action, Join),
|
|
270
258
|
'schema': action.param_class().model_json_schema(),
|
|
271
|
-
'
|
|
259
|
+
'actionOptions': action.action_options.json()
|
|
272
260
|
}
|
|
273
261
|
|
|
274
262
|
@staticmethod
|
|
@@ -337,10 +325,10 @@ class Plugin(object):
|
|
|
337
325
|
self.logger.info("Plugin starting")
|
|
338
326
|
|
|
339
327
|
for action in self.singleton_actions:
|
|
340
|
-
num_threads = 1
|
|
328
|
+
num_threads = 1
|
|
341
329
|
if self.action_name(action) in self.thread_config:
|
|
342
330
|
maybe_num_threads = self.thread_config[self.action_name(action)]
|
|
343
|
-
if
|
|
331
|
+
if maybe_num_threads is int and maybe_num_threads > 0:
|
|
344
332
|
num_threads = maybe_num_threads
|
|
345
333
|
else:
|
|
346
334
|
self.logger.error(f"Ignoring non-int or invalid thread value {maybe_num_threads}")
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
deltafi/__init__.py,sha256=FHwsfe4TgGeVLINN6940urXAzxGvl_Hof0gmOB32N7Y,709
|
|
2
|
-
deltafi/action.py,sha256=
|
|
2
|
+
deltafi/action.py,sha256=1axu4vgrmYVnIVzspCY04HG14eK3eGTKGg8h6uwyPk8,12749
|
|
3
3
|
deltafi/actioneventqueue.py,sha256=3f-ZClq2B1QsfNubvgnlvWT_Y-B5fvER9TDrMkJkfmQ,3005
|
|
4
4
|
deltafi/actiontype.py,sha256=2FnTbryP2zVlJfghrh9IqhIHWarPfSKSKt2UxoIrGLU,913
|
|
5
5
|
deltafi/domain.py,sha256=4tj5DFw6nsSZo9KG5LmUb3-IETRzth_wOt38VVT74HE,13720
|
|
@@ -8,7 +8,7 @@ deltafi/genericmodel.py,sha256=0qgii2o-fzHS6lM1C5X7_WTTSYsiZAM0i7RTfw0nw4U,1152
|
|
|
8
8
|
deltafi/input.py,sha256=CZ02GsObhu6CpdBKzNE5TJ_PSihQF3cr4XKPqouqRUU,1780
|
|
9
9
|
deltafi/logger.py,sha256=Lms9wmjyZWe9-SCKj4raNmGC9Cyn3BrfiniPXeM9VDY,2140
|
|
10
10
|
deltafi/metric.py,sha256=5Q40bLpNXk8mwu1wnN5L8GX22-iZxYEFBGt3zSNmYxI,972
|
|
11
|
-
deltafi/plugin.py,sha256=
|
|
11
|
+
deltafi/plugin.py,sha256=sGpGhsOZXojrXx6UOZxu8VhUBsIALpLUZaywBTj_WMg,19082
|
|
12
12
|
deltafi/result.py,sha256=j_3icJ74xIQo9KQV738XiqaFOYirjuIgFEtld4pBvXk,9555
|
|
13
13
|
deltafi/storage.py,sha256=zgBkaDTiT4BLEE1lAYjq4mqS4zqZKzuoVmG1CNbkD20,2979
|
|
14
14
|
deltafi/test_kit/__init__.py,sha256=FHwsfe4TgGeVLINN6940urXAzxGvl_Hof0gmOB32N7Y,709
|
|
@@ -19,6 +19,6 @@ deltafi/test_kit/egress.py,sha256=MXEsYsgiG1G-1zCuwV23RCn7w5bPObP1J0_7nVN03MU,18
|
|
|
19
19
|
deltafi/test_kit/framework.py,sha256=pU8tLsMNWUhVtIhLusbQL3uPT4y8xsKWdzzXuxmxRrU,15432
|
|
20
20
|
deltafi/test_kit/timed_ingress.py,sha256=ogk6AzbPZInLOPMYy3MhjEXpITPn9-Ha6YSILGBdpro,4177
|
|
21
21
|
deltafi/test_kit/transform.py,sha256=7KOsNY9OSDiyTFXIx0knZeweXIA421vHS6PAxQAcOc0,4122
|
|
22
|
-
deltafi-2.
|
|
23
|
-
deltafi-2.
|
|
24
|
-
deltafi-2.
|
|
22
|
+
deltafi-2.15.0.dist-info/METADATA,sha256=Bqa4ypyIaqu3wrNVKkRMWvZmyp4SFOAnI6ymu2KJoX4,1519
|
|
23
|
+
deltafi-2.15.0.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
|
|
24
|
+
deltafi-2.15.0.dist-info/RECORD,,
|