dcicutils 8.5.0.1b3__py3-none-any.whl → 8.5.0.1b6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dcicutils/portal_utils.py +193 -112
- dcicutils/structured_data.py +20 -16
- {dcicutils-8.5.0.1b3.dist-info → dcicutils-8.5.0.1b6.dist-info}/METADATA +1 -1
- {dcicutils-8.5.0.1b3.dist-info → dcicutils-8.5.0.1b6.dist-info}/RECORD +7 -7
- {dcicutils-8.5.0.1b3.dist-info → dcicutils-8.5.0.1b6.dist-info}/LICENSE.txt +0 -0
- {dcicutils-8.5.0.1b3.dist-info → dcicutils-8.5.0.1b6.dist-info}/WHEEL +0 -0
- {dcicutils-8.5.0.1b3.dist-info → dcicutils-8.5.0.1b6.dist-info}/entry_points.txt +0 -0
dcicutils/portal_utils.py
CHANGED
@@ -1,19 +1,20 @@
|
|
1
1
|
from collections import deque
|
2
|
+
import io
|
3
|
+
import json
|
2
4
|
from pyramid.paster import get_app
|
3
5
|
from pyramid.router import Router
|
6
|
+
import os
|
4
7
|
import re
|
5
8
|
import requests
|
6
9
|
from requests.models import Response as RequestResponse
|
7
10
|
from typing import Optional, Type, Union
|
8
11
|
from webtest.app import TestApp, TestResponse
|
9
|
-
from dcicutils.common import OrchestratedApp,
|
10
|
-
from dcicutils.creds_utils import CGAPKeyManager, FourfrontKeyManager, SMaHTKeyManager
|
12
|
+
from dcicutils.common import OrchestratedApp, ORCHESTRATED_APPS
|
11
13
|
from dcicutils.ff_utils import get_metadata, get_schema, patch_metadata, post_metadata
|
12
14
|
from dcicutils.misc_utils import to_camel_case, VirtualApp
|
13
15
|
from dcicutils.zip_utils import temporary_file
|
14
16
|
|
15
17
|
Portal = Type["Portal"] # Forward type reference for type hints.
|
16
|
-
FILE_SCHEMA_NAME = "File"
|
17
18
|
|
18
19
|
|
19
20
|
class Portal:
|
@@ -23,115 +24,175 @@ class Portal:
|
|
23
24
|
2. From a key dictionary, containing "key" and "secret" property values.
|
24
25
|
3. From a key tuple, containing (in order) a key and secret values.
|
25
26
|
4. From a keys file assumed to reside in ~/.{app}-keys.json where the given "app" value is either "smaht", "cgap",
|
26
|
-
or "fourfront";
|
27
|
-
|
28
|
-
an "app" value is not specified but the given "env" value begins with one of the app values then that value
|
29
|
-
will be used, i.e. e.g. if env is "smaht-
|
27
|
+
or "fourfront"; where is assumed to contain a dictionary with a key for the given "env" value, e.g. smaht-local;
|
28
|
+
and with a dictionary value containing "key" and "secret" property values, and an optional "server" property;
|
29
|
+
if an "app" value is not specified but the given "env" value begins with one of the app values then that value
|
30
|
+
will be used, i.e. e.g. if "env" is "smaht-local" and app is unspecified than it is assumed to be "smaht".
|
30
31
|
5. From a keys file as described above (#4) but rather than be identified by the given "env" value it
|
31
|
-
is looked up
|
32
|
+
is looked up via the given "server" name and the "server" key dictionary value in the key file.
|
32
33
|
6. From a given "vapp" value (which is assumed to be a TestApp or VirtualApp).
|
33
|
-
7. From another Portal object.
|
34
|
-
8. From a a pyramid Router object.
|
34
|
+
7. From another Portal object; or from a a pyramid Router object.
|
35
35
|
"""
|
36
|
+
FILE_SCHEMA_NAME = "File"
|
37
|
+
KEYS_FILE_DIRECTORY = os.path.expanduser(f"~")
|
38
|
+
|
36
39
|
def __init__(self,
|
37
|
-
arg: Optional[Union[
|
38
|
-
env: Optional[str] = None,
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
self._vapp = portal._vapp
|
60
|
-
self._env = portal._env
|
61
|
-
self._app = portal._app
|
62
|
-
self._server = portal._server
|
40
|
+
arg: Optional[Union[Portal, TestApp, VirtualApp, Router, dict, tuple, str]] = None,
|
41
|
+
env: Optional[str] = None, server: Optional[str] = None,
|
42
|
+
app: Optional[OrchestratedApp] = None) -> None:
|
43
|
+
|
44
|
+
def init(unspecified: Optional[list] = []) -> None:
|
45
|
+
self._ini_file = None
|
46
|
+
self._key = None
|
47
|
+
self._key_pair = None
|
48
|
+
self._key_id = None
|
49
|
+
self._secret = None
|
50
|
+
self._keys_file = None
|
51
|
+
self._env = None
|
52
|
+
self._server = None
|
53
|
+
self._app = None
|
54
|
+
self._vapp = None
|
55
|
+
for arg in unspecified:
|
56
|
+
if arg is not None:
|
57
|
+
raise Exception("Portal init error; extraneous args.")
|
58
|
+
|
59
|
+
def init_from_portal(portal: Portal, unspecified: Optional[list] = None) -> None:
|
60
|
+
init(unspecified)
|
61
|
+
self._ini_file = portal._ini_file
|
63
62
|
self._key = portal._key
|
64
63
|
self._key_pair = portal._key_pair
|
65
64
|
self._key_id = portal._key_id
|
66
|
-
self.
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
self._key = None
|
73
|
-
self._key_pair = None
|
74
|
-
self._key_id = None
|
75
|
-
self._key_file = None
|
76
|
-
if isinstance(portal, (VirtualApp, TestApp)):
|
77
|
-
self._vapp = portal
|
78
|
-
elif isinstance(portal, (Router, str)):
|
79
|
-
self._vapp = Portal._create_vapp(portal)
|
80
|
-
elif isinstance(key, dict):
|
81
|
-
self._key = key
|
82
|
-
self._key_pair = (key.get("key"), key.get("secret")) if key else None
|
83
|
-
if key_server := key.get("server"):
|
84
|
-
self._server = key_server
|
85
|
-
elif isinstance(key, tuple) and len(key) >= 2:
|
86
|
-
self._key = {"key": key[0], "secret": key[1]}
|
87
|
-
self._key_pair = key
|
88
|
-
elif isinstance(env, str):
|
89
|
-
key_managers = {APP_CGAP: CGAPKeyManager, APP_FOURFRONT: FourfrontKeyManager, APP_SMAHT: SMaHTKeyManager}
|
90
|
-
if not (key_manager := key_managers.get(self._app)) or not (key_manager := key_manager()):
|
91
|
-
raise Exception(f"Invalid app name: {self._app} (valid: {', '.join(ORCHESTRATED_APPS)}).")
|
92
|
-
if isinstance(env, str):
|
93
|
-
self._key = key_manager.get_keydict_for_env(env)
|
94
|
-
if key_server := self._key.get("server"):
|
95
|
-
self._server = key_server
|
96
|
-
elif isinstance(self._server, str):
|
97
|
-
self._key = key_manager.get_keydict_for_server(self._server)
|
98
|
-
self._key_pair = key_manager.keydict_to_keypair(self._key) if self._key else None
|
99
|
-
self._key_file = key_manager.keys_file
|
100
|
-
if self._key and (key_id := self._key.get("key")):
|
101
|
-
self._key_id = key_id
|
102
|
-
elif self._key_pair and (key_id := self._key_pair[1]):
|
103
|
-
self._key_id = key_id
|
65
|
+
self._secret = portal._secret
|
66
|
+
self._keys_file = portal._keys_file
|
67
|
+
self._env = portal._env
|
68
|
+
self._server = portal._server
|
69
|
+
self._app = portal._app
|
70
|
+
self._vapp = portal._vapp
|
104
71
|
|
105
|
-
|
106
|
-
|
107
|
-
|
72
|
+
def init_from_vapp(vapp: Union[TestApp, VirtualApp, Router], unspecified: Optional[list] = []) -> None:
|
73
|
+
init(unspecified)
|
74
|
+
self._vapp = Portal._create_testapp(vapp)
|
108
75
|
|
109
|
-
|
110
|
-
|
111
|
-
|
76
|
+
def init_from_ini_file(ini_file: str, unspecified: Optional[list] = []) -> None:
|
77
|
+
init(unspecified)
|
78
|
+
self._ini_file = ini_file
|
79
|
+
self._vapp = Portal._create_testapp(ini_file)
|
80
|
+
|
81
|
+
def init_from_key(key: dict, server: Optional[str], unspecified: Optional[list] = []) -> None:
|
82
|
+
init(unspecified)
|
83
|
+
if (isinstance(key_id := key.get("key"), str) and key_id and
|
84
|
+
isinstance(secret := key.get("secret"), str) and secret): # noqa
|
85
|
+
self._key = {"key": key_id, "secret": secret}
|
86
|
+
self._key_id = key_id
|
87
|
+
self._secret = secret
|
88
|
+
self._key_pair = (key_id, secret)
|
89
|
+
if ((isinstance(server, str) and server) or (isinstance(server := key.get("server"), str) and server)):
|
90
|
+
if server := normalize_server(server):
|
91
|
+
self._key["server"] = self._server = server
|
92
|
+
if not self._key:
|
93
|
+
raise Exception("Portal init error; from key.")
|
94
|
+
|
95
|
+
def init_from_key_pair(key_pair: tuple, server: Optional[str], unspecified: Optional[list] = []) -> None:
|
96
|
+
if len(key_pair) == 2:
|
97
|
+
init_from_key({"key": key_pair[0], "secret": key_pair[1]}, server, unspecified)
|
98
|
+
else:
|
99
|
+
raise Exception("Portal init error; from key-pair.")
|
100
|
+
|
101
|
+
def init_from_keys_file(keys_file: str, env: Optional[str], server: Optional[str],
|
102
|
+
unspecified: Optional[list] = []) -> None:
|
103
|
+
try:
|
104
|
+
with io.open(keys_file) as f:
|
105
|
+
keys = json.load(f)
|
106
|
+
except Exception:
|
107
|
+
raise Exception(f"Portal init error; cannot open keys-file: {keys_file}")
|
108
|
+
if isinstance(env, str) and env and isinstance(key := keys.get(env), dict):
|
109
|
+
init_from_key(key, server)
|
110
|
+
self._keys_file = keys_file
|
111
|
+
self._env = env
|
112
|
+
elif isinstance(server, str) and server and (key := [k for k in keys if keys[k].get("server") == server]):
|
113
|
+
init_from_key(key, server)
|
114
|
+
self._keys_file = keys_file
|
115
|
+
elif len(keys) == 1 and (env := next(iter(keys))) and isinstance(key := keys[env], dict) and key:
|
116
|
+
init_from_key(key, server)
|
117
|
+
self._keys_file = keys_file
|
118
|
+
self._env = env
|
119
|
+
else:
|
120
|
+
raise Exception(f"Portal init error; {env or server or None} not found in keys-file: {keys_file}")
|
121
|
+
|
122
|
+
def init_from_env_server_app(env: str, server: str, app: Optional[str],
|
123
|
+
unspecified: Optional[list] = None) -> None:
|
124
|
+
return init_from_keys_file(self._default_keys_file(app, env), env, server, unspecified)
|
125
|
+
|
126
|
+
def normalize_server(server: str) -> Optional[str]:
|
127
|
+
prefix = ""
|
128
|
+
if (lserver := server.lower()).startswith("http://"):
|
129
|
+
prefix = "http://"
|
130
|
+
elif lserver.startswith("https://"):
|
131
|
+
prefix = "https://"
|
132
|
+
if prefix:
|
133
|
+
if (server := re.sub(r"/+", "/", server[len(prefix):])).startswith("/"):
|
134
|
+
server = server[1:]
|
135
|
+
if len(server) > 1 and server.endswith("/"):
|
136
|
+
server = server[:-1]
|
137
|
+
return prefix + server if server else None
|
138
|
+
|
139
|
+
if isinstance(arg, Portal):
|
140
|
+
init_from_portal(arg, unspecified=[env, server, app])
|
141
|
+
elif isinstance(arg, (TestApp, VirtualApp, Router)):
|
142
|
+
init_from_vapp(arg, unspecified=[env, server, app])
|
143
|
+
elif isinstance(arg, str) and arg.endswith(".ini"):
|
144
|
+
init_from_ini_file(arg, unspecified=[env, server, app])
|
145
|
+
elif isinstance(arg, dict):
|
146
|
+
init_from_key(arg, server, unspecified=[env, app])
|
147
|
+
elif isinstance(arg, tuple):
|
148
|
+
init_from_key_pair(arg, server, unspecified=[env, app])
|
149
|
+
elif isinstance(arg, str) and arg.endswith(".json"):
|
150
|
+
init_from_keys_file(arg, env, server, unspecified=[app])
|
151
|
+
elif isinstance(arg, str) and arg:
|
152
|
+
init_from_env_server_app(arg, server, app, unspecified=[env])
|
153
|
+
elif isinstance(env, str) and env:
|
154
|
+
init_from_env_server_app(env, server, app, unspecified=[arg])
|
155
|
+
else:
|
156
|
+
raise Exception("Portal init error; invalid args.")
|
112
157
|
|
113
158
|
@property
|
114
|
-
def
|
115
|
-
return self.
|
159
|
+
def ini_file(self) -> Optional[str]:
|
160
|
+
return self._ini_file
|
116
161
|
|
117
162
|
@property
|
118
|
-
def key(self):
|
163
|
+
def key(self) -> Optional[dict]:
|
119
164
|
return self._key
|
120
165
|
|
121
166
|
@property
|
122
|
-
def key_pair(self):
|
167
|
+
def key_pair(self) -> Optional[tuple]:
|
123
168
|
return self._key_pair
|
124
169
|
|
125
170
|
@property
|
126
|
-
def key_id(self):
|
171
|
+
def key_id(self) -> Optional[str]:
|
127
172
|
return self._key_id
|
128
173
|
|
129
174
|
@property
|
130
|
-
def
|
131
|
-
return self.
|
175
|
+
def secret(self) -> Optional[str]:
|
176
|
+
return self._secret
|
177
|
+
|
178
|
+
@property
|
179
|
+
def keys_file(self) -> Optional[str]:
|
180
|
+
return self._keys_file
|
181
|
+
|
182
|
+
@property
|
183
|
+
def env(self) -> Optional[str]:
|
184
|
+
return self._env
|
185
|
+
|
186
|
+
@property
|
187
|
+
def server(self) -> Optional[str]:
|
188
|
+
return self._server
|
132
189
|
|
133
190
|
@property
|
134
|
-
def
|
191
|
+
def app(self) -> Optional[str]:
|
192
|
+
return self._app
|
193
|
+
|
194
|
+
@property
|
195
|
+
def vapp(self) -> Optional[TestApp]:
|
135
196
|
return self._vapp
|
136
197
|
|
137
198
|
def get_metadata(self, object_id: str) -> Optional[dict]:
|
@@ -148,27 +209,27 @@ class Portal:
|
|
148
209
|
return self.post(f"/{object_type}", data)
|
149
210
|
|
150
211
|
def get(self, uri: str, follow: bool = True, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
|
151
|
-
if
|
152
|
-
response = self._vapp.get(self.
|
212
|
+
if self._vapp:
|
213
|
+
response = self._vapp.get(self.url(uri), **self._kwargs(**kwargs))
|
153
214
|
if response and response.status_code in [301, 302, 303, 307, 308] and follow:
|
154
215
|
response = response.follow()
|
155
216
|
return self._response(response)
|
156
|
-
return requests.get(self.
|
217
|
+
return requests.get(self.url(uri), allow_redirects=follow, **self._kwargs(**kwargs))
|
157
218
|
|
158
219
|
def patch(self, uri: str, data: Optional[dict] = None,
|
159
220
|
json: Optional[dict] = None, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
|
160
|
-
if
|
161
|
-
return self._vapp.patch_json(self.
|
162
|
-
return requests.patch(self.
|
221
|
+
if self._vapp:
|
222
|
+
return self._vapp.patch_json(self.url(uri), json or data, **self._kwargs(**kwargs))
|
223
|
+
return requests.patch(self.url(uri), data=data, json=json, **self._kwargs(**kwargs))
|
163
224
|
|
164
225
|
def post(self, uri: str, data: Optional[dict] = None, json: Optional[dict] = None,
|
165
226
|
files: Optional[dict] = None, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
|
166
|
-
if
|
227
|
+
if self._vapp:
|
167
228
|
if files:
|
168
|
-
return self._vapp.post(self.
|
229
|
+
return self._vapp.post(self.url(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
|
169
230
|
else:
|
170
|
-
return self._vapp.post_json(self.
|
171
|
-
return requests.post(self.
|
231
|
+
return self._vapp.post_json(self.url(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
|
232
|
+
return requests.post(self.url(uri), data=data, json=json, files=files, **self._kwargs(**kwargs))
|
172
233
|
|
173
234
|
def get_schema(self, schema_name: str) -> Optional[dict]:
|
174
235
|
return get_schema(self.schema_name(schema_name), portal_vapp=self._vapp, key=self._key)
|
@@ -182,7 +243,7 @@ class Portal:
|
|
182
243
|
|
183
244
|
def is_file_schema(self, schema_name: str) -> bool:
|
184
245
|
if super_type_map := self.get_schemas_super_type_map():
|
185
|
-
if file_super_type := super_type_map.get(FILE_SCHEMA_NAME):
|
246
|
+
if file_super_type := super_type_map.get(Portal.FILE_SCHEMA_NAME):
|
186
247
|
return self.schema_name(schema_name) in file_super_type
|
187
248
|
return False
|
188
249
|
|
@@ -222,13 +283,14 @@ class Portal:
|
|
222
283
|
except Exception:
|
223
284
|
return False
|
224
285
|
|
225
|
-
def
|
286
|
+
def url(self, uri: str) -> str:
|
226
287
|
if not isinstance(uri, str) or not uri:
|
227
288
|
return "/"
|
228
|
-
if uri.lower().startswith("http://") or
|
289
|
+
if (luri := uri.lower()).startswith("http://") or luri.startswith("https://"):
|
229
290
|
return uri
|
230
|
-
uri
|
231
|
-
|
291
|
+
if not (uri := re.sub(r"/+", "/", uri)).startswith("/"):
|
292
|
+
uri = "/"
|
293
|
+
return self._server + uri if self._server else uri
|
232
294
|
|
233
295
|
def _kwargs(self, **kwargs) -> dict:
|
234
296
|
result_kwargs = {"headers":
|
@@ -239,6 +301,16 @@ class Portal:
|
|
239
301
|
result_kwargs["timeout"] = timeout
|
240
302
|
return result_kwargs
|
241
303
|
|
304
|
+
def _default_keys_file(self, app: Optional[str], env: Optional[str] = None) -> Optional[str]:
|
305
|
+
def is_valid_app(app: Optional[str]) -> bool: # noqa
|
306
|
+
return app and app.lower() in [name.lower() for name in ORCHESTRATED_APPS]
|
307
|
+
def infer_app_from_env(env: str) -> Optional[str]: # noqa
|
308
|
+
if isinstance(env, str) and (lenv := env.lower()):
|
309
|
+
if app := [app for app in ORCHESTRATED_APPS if lenv.startswith(app.lower())]:
|
310
|
+
return app[0]
|
311
|
+
if is_valid_app(app) or (app := infer_app_from_env(env)):
|
312
|
+
return os.path.join(Portal.KEYS_FILE_DIRECTORY, f".{app.lower()}-keys.json")
|
313
|
+
|
242
314
|
def _response(self, response) -> Optional[RequestResponse]:
|
243
315
|
if response and isinstance(getattr(response.__class__, "json"), property):
|
244
316
|
class RequestResponseWrapper: # For consistency change json property to method.
|
@@ -255,15 +327,15 @@ class Portal:
|
|
255
327
|
@staticmethod
|
256
328
|
def create_for_testing(ini_file: Optional[str] = None) -> Portal:
|
257
329
|
if isinstance(ini_file, str):
|
258
|
-
return Portal(Portal.
|
330
|
+
return Portal(Portal._create_testapp(ini_file))
|
259
331
|
minimal_ini_for_unit_testing = "[app:app]\nuse = egg:encoded\nsqlalchemy.url = postgresql://dummy\n"
|
260
332
|
with temporary_file(content=minimal_ini_for_unit_testing, suffix=".ini") as ini_file:
|
261
|
-
return Portal(Portal.
|
333
|
+
return Portal(Portal._create_testapp(ini_file))
|
262
334
|
|
263
335
|
@staticmethod
|
264
336
|
def create_for_testing_local(ini_file: Optional[str] = None) -> Portal:
|
265
337
|
if isinstance(ini_file, str) and ini_file:
|
266
|
-
return Portal(Portal.
|
338
|
+
return Portal(Portal._create_testapp(ini_file))
|
267
339
|
minimal_ini_for_testing_local = "\n".join([
|
268
340
|
"[app:app]\nuse = egg:encoded\nfile_upload_bucket = dummy",
|
269
341
|
"sqlalchemy.url = postgresql://postgres@localhost:5441/postgres?host=/tmp/snovault/pgdata",
|
@@ -284,11 +356,20 @@ class Portal:
|
|
284
356
|
"multiauth.policy.auth0.base = encoded.authentication.Auth0AuthenticationPolicy"
|
285
357
|
])
|
286
358
|
with temporary_file(content=minimal_ini_for_testing_local, suffix=".ini") as minimal_ini_file:
|
287
|
-
return Portal(Portal.
|
359
|
+
return Portal(Portal._create_testapp(minimal_ini_file))
|
288
360
|
|
289
361
|
@staticmethod
|
290
|
-
def
|
291
|
-
if isinstance(
|
292
|
-
return
|
293
|
-
|
294
|
-
|
362
|
+
def _create_testapp(arg: Union[TestApp, VirtualApp, Router, str] = None, app_name: Optional[str] = None) -> TestApp:
|
363
|
+
if isinstance(arg, TestApp):
|
364
|
+
return arg
|
365
|
+
elif isinstance(arg, VirtualApp):
|
366
|
+
if not isinstance(arg.wrapped_app, TestApp):
|
367
|
+
raise Exception("Portal._create_testapp VirtualApp argument error.")
|
368
|
+
return arg.wrapped_app
|
369
|
+
if isinstance(arg, Router):
|
370
|
+
router = arg
|
371
|
+
elif isinstance(arg, str) or arg is None:
|
372
|
+
router = get_app(arg or "development.ini", app_name or "app")
|
373
|
+
else:
|
374
|
+
raise Exception("Portal._create_testapp argument error.")
|
375
|
+
return TestApp(router, {"HTTP_ACCEPT": "application/json", "REMOTE_USER": "TEST"})
|
dcicutils/structured_data.py
CHANGED
@@ -42,9 +42,9 @@ StructuredDataSet = Type["StructuredDataSet"]
|
|
42
42
|
class StructuredDataSet:
|
43
43
|
|
44
44
|
def __init__(self, file: Optional[str] = None, portal: Optional[Union[VirtualApp, TestApp, Portal]] = None,
|
45
|
-
schemas: Optional[List[dict]] = None,
|
45
|
+
schemas: Optional[List[dict]] = None, autoadd: Optional[dict] = None,
|
46
46
|
order: Optional[List[str]] = None, prune: bool = True) -> None:
|
47
|
-
self.data = {}
|
47
|
+
self.data = {}
|
48
48
|
self._portal = Portal(portal, data=self.data, schemas=schemas) if portal else None
|
49
49
|
self._order = order
|
50
50
|
self._prune = prune
|
@@ -52,13 +52,14 @@ class StructuredDataSet:
|
|
52
52
|
self._errors = {}
|
53
53
|
self._resolved_refs = set()
|
54
54
|
self._validated = False
|
55
|
+
self._autoadd_properties = autoadd if isinstance(autoadd, dict) and autoadd else None
|
55
56
|
self._load_file(file) if file else None
|
56
57
|
|
57
58
|
@staticmethod
|
58
59
|
def load(file: str, portal: Optional[Union[VirtualApp, TestApp, Portal]] = None,
|
59
|
-
schemas: Optional[List[dict]] = None,
|
60
|
+
schemas: Optional[List[dict]] = None, autoadd: Optional[dict] = None,
|
60
61
|
order: Optional[List[str]] = None, prune: bool = True) -> StructuredDataSet:
|
61
|
-
return StructuredDataSet(file=file, portal=portal, schemas=schemas, order=order, prune=prune)
|
62
|
+
return StructuredDataSet(file=file, portal=portal, schemas=schemas, autoadd=autoadd, order=order, prune=prune)
|
62
63
|
|
63
64
|
def validate(self, force: bool = False) -> None:
|
64
65
|
if self._validated and not force:
|
@@ -112,7 +113,7 @@ class StructuredDataSet:
|
|
112
113
|
def _load_file(self, file: str) -> None:
|
113
114
|
# Returns a dictionary where each property is the name (i.e. the type) of the data,
|
114
115
|
# and the value is array of dictionaries for the data itself. Handle these kinds of files:
|
115
|
-
# 1. Single CSV
|
116
|
+
# 1. Single CSV, TSV, or JSON file, where the (base) name of the file is the data type name.
|
116
117
|
# 2. Single Excel file containing one or more sheets, where each sheet
|
117
118
|
# represents (i.e. is named for, and contains data for) a different type.
|
118
119
|
# 3. Zip file (.zip or .tar.gz or .tgz or .tar), containing data files to load, where the
|
@@ -163,6 +164,8 @@ class StructuredDataSet:
|
|
163
164
|
structured_row = structured_row_template.create_row()
|
164
165
|
for column_name, value in row.items():
|
165
166
|
structured_row_template.set_value(structured_row, column_name, value, reader.file, reader.row_number)
|
167
|
+
if self._autoadd_properties:
|
168
|
+
self._add_properties(structured_row, self._autoadd_properties, schema)
|
166
169
|
self._add(type_name, structured_row)
|
167
170
|
self._note_warning(reader.warnings, "reader")
|
168
171
|
if schema:
|
@@ -177,6 +180,11 @@ class StructuredDataSet:
|
|
177
180
|
else:
|
178
181
|
self.data[type_name] = [data] if isinstance(data, dict) else data
|
179
182
|
|
183
|
+
def _add_properties(self, structured_row: dict, properties: dict, schema: Optional[dict] = None) -> None:
|
184
|
+
for name in properties:
|
185
|
+
if name not in structured_row and (not schema or schema.data.get("properties", {}).get(name)):
|
186
|
+
structured_row[name] = properties[name]
|
187
|
+
|
180
188
|
def _note_warning(self, item: Optional[Union[dict, List[dict]]], group: str) -> None:
|
181
189
|
self._note_issue(self._warnings, item, group)
|
182
190
|
|
@@ -475,7 +483,6 @@ class Schema:
|
|
475
483
|
if unique:
|
476
484
|
typeinfo[key]["unique"] = True
|
477
485
|
result.update(typeinfo)
|
478
|
-
# result.update(self._create_typeinfo(array_property_items, parent_key=key))
|
479
486
|
continue
|
480
487
|
result[key] = {"type": property_value_type, "map": self._map_function({**property_value, "column": key})}
|
481
488
|
if ARRAY_NAME_SUFFIX_CHAR in key:
|
@@ -543,16 +550,13 @@ class Portal(PortalBase):
|
|
543
550
|
|
544
551
|
def __init__(self,
|
545
552
|
arg: Optional[Union[VirtualApp, TestApp, Router, Portal, dict, tuple, str]] = None,
|
546
|
-
env: Optional[str] = None,
|
547
|
-
|
548
|
-
|
549
|
-
|
550
|
-
|
551
|
-
|
552
|
-
|
553
|
-
if isinstance(portal, Portal):
|
554
|
-
self._schemas = schemas if schemas is not None else portal._schemas # Explicitly specified/known schemas.
|
555
|
-
self._data = data if data is not None else portal._data # Data set being loaded; e.g. by StructuredDataSet.
|
553
|
+
env: Optional[str] = None, server: Optional[str] = None,
|
554
|
+
app: Optional[OrchestratedApp] = None,
|
555
|
+
data: Optional[dict] = None, schemas: Optional[List[dict]] = None) -> None:
|
556
|
+
super().__init__(arg, env=env, server=server, app=app)
|
557
|
+
if isinstance(arg, Portal):
|
558
|
+
self._schemas = schemas if schemas is not None else arg._schemas # Explicitly specified/known schemas.
|
559
|
+
self._data = data if data is not None else arg._data # Data set being loaded; e.g. by StructuredDataSet.
|
556
560
|
else:
|
557
561
|
self._schemas = schemas
|
558
562
|
self._data = data
|
@@ -43,7 +43,7 @@ dcicutils/log_utils.py,sha256=7pWMc6vyrorUZQf-V-M3YC6zrPgNhuV_fzm9xqTPph0,10883
|
|
43
43
|
dcicutils/misc_utils.py,sha256=nRjLEORY35YmJwTjO0fnauBPznaI_bkVasIW8PccDYM,100179
|
44
44
|
dcicutils/obfuscation_utils.py,sha256=fo2jOmDRC6xWpYX49u80bVNisqRRoPskFNX3ymFAmjw,5963
|
45
45
|
dcicutils/opensearch_utils.py,sha256=V2exmFYW8Xl2_pGFixF4I2Cc549Opwe4PhFi5twC0M8,1017
|
46
|
-
dcicutils/portal_utils.py,sha256=
|
46
|
+
dcicutils/portal_utils.py,sha256=PLIKONo_BhVrf-r5fkXAaxP5IQZRjvPjf83YxvRx0ZE,18393
|
47
47
|
dcicutils/project_utils.py,sha256=qPdCaFmWUVBJw4rw342iUytwdQC0P-XKpK4mhyIulMM,31250
|
48
48
|
dcicutils/qa_checkers.py,sha256=cdXjeL0jCDFDLT8VR8Px78aS10hwNISOO5G_Zv2TZ6M,20534
|
49
49
|
dcicutils/qa_utils.py,sha256=TT0SiJWiuxYvbsIyhK9VO4uV_suxhB6CpuC4qPacCzQ,160208
|
@@ -56,14 +56,14 @@ dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19
|
|
56
56
|
dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
|
57
57
|
dcicutils/snapshot_utils.py,sha256=ymP7PXH6-yEiXAt75w0ldQFciGNqWBClNxC5gfX2FnY,22961
|
58
58
|
dcicutils/ssl_certificate_utils.py,sha256=F0ifz_wnRRN9dfrfsz7aCp4UDLgHEY8LaK7PjnNvrAQ,9707
|
59
|
-
dcicutils/structured_data.py,sha256=
|
59
|
+
dcicutils/structured_data.py,sha256=mWjH6h9ARUvYMHI4ZiwbaoKtoeM0CAOz1UxCHh4HIHE,32874
|
60
60
|
dcicutils/task_utils.py,sha256=MF8ujmTD6-O2AC2gRGPHyGdUrVKgtr8epT5XU8WtNjk,8082
|
61
61
|
dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
|
62
62
|
dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
|
63
63
|
dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
|
64
64
|
dcicutils/zip_utils.py,sha256=0OXR0aLNwyLIZOzIFTM_5DOun7dxIv6TIZbFiithkO0,3276
|
65
|
-
dcicutils-8.5.0.
|
66
|
-
dcicutils-8.5.0.
|
67
|
-
dcicutils-8.5.0.
|
68
|
-
dcicutils-8.5.0.
|
69
|
-
dcicutils-8.5.0.
|
65
|
+
dcicutils-8.5.0.1b6.dist-info/LICENSE.txt,sha256=t0_-jIjqxNnymZoNJe-OltRIuuF8qfhN0ATlHyrUJPk,1102
|
66
|
+
dcicutils-8.5.0.1b6.dist-info/METADATA,sha256=wWiZixknI6Pne33sGS-Lxeh6hSKSScj44tPcctlpHJw,3314
|
67
|
+
dcicutils-8.5.0.1b6.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
|
68
|
+
dcicutils-8.5.0.1b6.dist-info/entry_points.txt,sha256=8wbw5csMIgBXhkwfgsgJeuFcoUc0WsucUxmOyml2aoA,209
|
69
|
+
dcicutils-8.5.0.1b6.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|