dcicutils 8.5.0.1b3__py3-none-any.whl → 8.5.0.1b6__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- dcicutils/portal_utils.py +193 -112
- dcicutils/structured_data.py +20 -16
- {dcicutils-8.5.0.1b3.dist-info → dcicutils-8.5.0.1b6.dist-info}/METADATA +1 -1
- {dcicutils-8.5.0.1b3.dist-info → dcicutils-8.5.0.1b6.dist-info}/RECORD +7 -7
- {dcicutils-8.5.0.1b3.dist-info → dcicutils-8.5.0.1b6.dist-info}/LICENSE.txt +0 -0
- {dcicutils-8.5.0.1b3.dist-info → dcicutils-8.5.0.1b6.dist-info}/WHEEL +0 -0
- {dcicutils-8.5.0.1b3.dist-info → dcicutils-8.5.0.1b6.dist-info}/entry_points.txt +0 -0
dcicutils/portal_utils.py
CHANGED
@@ -1,19 +1,20 @@
|
|
1
1
|
from collections import deque
|
2
|
+
import io
|
3
|
+
import json
|
2
4
|
from pyramid.paster import get_app
|
3
5
|
from pyramid.router import Router
|
6
|
+
import os
|
4
7
|
import re
|
5
8
|
import requests
|
6
9
|
from requests.models import Response as RequestResponse
|
7
10
|
from typing import Optional, Type, Union
|
8
11
|
from webtest.app import TestApp, TestResponse
|
9
|
-
from dcicutils.common import OrchestratedApp,
|
10
|
-
from dcicutils.creds_utils import CGAPKeyManager, FourfrontKeyManager, SMaHTKeyManager
|
12
|
+
from dcicutils.common import OrchestratedApp, ORCHESTRATED_APPS
|
11
13
|
from dcicutils.ff_utils import get_metadata, get_schema, patch_metadata, post_metadata
|
12
14
|
from dcicutils.misc_utils import to_camel_case, VirtualApp
|
13
15
|
from dcicutils.zip_utils import temporary_file
|
14
16
|
|
15
17
|
Portal = Type["Portal"] # Forward type reference for type hints.
|
16
|
-
FILE_SCHEMA_NAME = "File"
|
17
18
|
|
18
19
|
|
19
20
|
class Portal:
|
@@ -23,115 +24,175 @@ class Portal:
|
|
23
24
|
2. From a key dictionary, containing "key" and "secret" property values.
|
24
25
|
3. From a key tuple, containing (in order) a key and secret values.
|
25
26
|
4. From a keys file assumed to reside in ~/.{app}-keys.json where the given "app" value is either "smaht", "cgap",
|
26
|
-
or "fourfront";
|
27
|
-
|
28
|
-
an "app" value is not specified but the given "env" value begins with one of the app values then that value
|
29
|
-
will be used, i.e. e.g. if env is "smaht-
|
27
|
+
or "fourfront"; where is assumed to contain a dictionary with a key for the given "env" value, e.g. smaht-local;
|
28
|
+
and with a dictionary value containing "key" and "secret" property values, and an optional "server" property;
|
29
|
+
if an "app" value is not specified but the given "env" value begins with one of the app values then that value
|
30
|
+
will be used, i.e. e.g. if "env" is "smaht-local" and app is unspecified than it is assumed to be "smaht".
|
30
31
|
5. From a keys file as described above (#4) but rather than be identified by the given "env" value it
|
31
|
-
is looked up
|
32
|
+
is looked up via the given "server" name and the "server" key dictionary value in the key file.
|
32
33
|
6. From a given "vapp" value (which is assumed to be a TestApp or VirtualApp).
|
33
|
-
7. From another Portal object.
|
34
|
-
8. From a a pyramid Router object.
|
34
|
+
7. From another Portal object; or from a a pyramid Router object.
|
35
35
|
"""
|
36
|
+
FILE_SCHEMA_NAME = "File"
|
37
|
+
KEYS_FILE_DIRECTORY = os.path.expanduser(f"~")
|
38
|
+
|
36
39
|
def __init__(self,
|
37
|
-
arg: Optional[Union[
|
38
|
-
env: Optional[str] = None,
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
self._vapp = portal._vapp
|
60
|
-
self._env = portal._env
|
61
|
-
self._app = portal._app
|
62
|
-
self._server = portal._server
|
40
|
+
arg: Optional[Union[Portal, TestApp, VirtualApp, Router, dict, tuple, str]] = None,
|
41
|
+
env: Optional[str] = None, server: Optional[str] = None,
|
42
|
+
app: Optional[OrchestratedApp] = None) -> None:
|
43
|
+
|
44
|
+
def init(unspecified: Optional[list] = []) -> None:
|
45
|
+
self._ini_file = None
|
46
|
+
self._key = None
|
47
|
+
self._key_pair = None
|
48
|
+
self._key_id = None
|
49
|
+
self._secret = None
|
50
|
+
self._keys_file = None
|
51
|
+
self._env = None
|
52
|
+
self._server = None
|
53
|
+
self._app = None
|
54
|
+
self._vapp = None
|
55
|
+
for arg in unspecified:
|
56
|
+
if arg is not None:
|
57
|
+
raise Exception("Portal init error; extraneous args.")
|
58
|
+
|
59
|
+
def init_from_portal(portal: Portal, unspecified: Optional[list] = None) -> None:
|
60
|
+
init(unspecified)
|
61
|
+
self._ini_file = portal._ini_file
|
63
62
|
self._key = portal._key
|
64
63
|
self._key_pair = portal._key_pair
|
65
64
|
self._key_id = portal._key_id
|
66
|
-
self.
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
self._key = None
|
73
|
-
self._key_pair = None
|
74
|
-
self._key_id = None
|
75
|
-
self._key_file = None
|
76
|
-
if isinstance(portal, (VirtualApp, TestApp)):
|
77
|
-
self._vapp = portal
|
78
|
-
elif isinstance(portal, (Router, str)):
|
79
|
-
self._vapp = Portal._create_vapp(portal)
|
80
|
-
elif isinstance(key, dict):
|
81
|
-
self._key = key
|
82
|
-
self._key_pair = (key.get("key"), key.get("secret")) if key else None
|
83
|
-
if key_server := key.get("server"):
|
84
|
-
self._server = key_server
|
85
|
-
elif isinstance(key, tuple) and len(key) >= 2:
|
86
|
-
self._key = {"key": key[0], "secret": key[1]}
|
87
|
-
self._key_pair = key
|
88
|
-
elif isinstance(env, str):
|
89
|
-
key_managers = {APP_CGAP: CGAPKeyManager, APP_FOURFRONT: FourfrontKeyManager, APP_SMAHT: SMaHTKeyManager}
|
90
|
-
if not (key_manager := key_managers.get(self._app)) or not (key_manager := key_manager()):
|
91
|
-
raise Exception(f"Invalid app name: {self._app} (valid: {', '.join(ORCHESTRATED_APPS)}).")
|
92
|
-
if isinstance(env, str):
|
93
|
-
self._key = key_manager.get_keydict_for_env(env)
|
94
|
-
if key_server := self._key.get("server"):
|
95
|
-
self._server = key_server
|
96
|
-
elif isinstance(self._server, str):
|
97
|
-
self._key = key_manager.get_keydict_for_server(self._server)
|
98
|
-
self._key_pair = key_manager.keydict_to_keypair(self._key) if self._key else None
|
99
|
-
self._key_file = key_manager.keys_file
|
100
|
-
if self._key and (key_id := self._key.get("key")):
|
101
|
-
self._key_id = key_id
|
102
|
-
elif self._key_pair and (key_id := self._key_pair[1]):
|
103
|
-
self._key_id = key_id
|
65
|
+
self._secret = portal._secret
|
66
|
+
self._keys_file = portal._keys_file
|
67
|
+
self._env = portal._env
|
68
|
+
self._server = portal._server
|
69
|
+
self._app = portal._app
|
70
|
+
self._vapp = portal._vapp
|
104
71
|
|
105
|
-
|
106
|
-
|
107
|
-
|
72
|
+
def init_from_vapp(vapp: Union[TestApp, VirtualApp, Router], unspecified: Optional[list] = []) -> None:
|
73
|
+
init(unspecified)
|
74
|
+
self._vapp = Portal._create_testapp(vapp)
|
108
75
|
|
109
|
-
|
110
|
-
|
111
|
-
|
76
|
+
def init_from_ini_file(ini_file: str, unspecified: Optional[list] = []) -> None:
|
77
|
+
init(unspecified)
|
78
|
+
self._ini_file = ini_file
|
79
|
+
self._vapp = Portal._create_testapp(ini_file)
|
80
|
+
|
81
|
+
def init_from_key(key: dict, server: Optional[str], unspecified: Optional[list] = []) -> None:
|
82
|
+
init(unspecified)
|
83
|
+
if (isinstance(key_id := key.get("key"), str) and key_id and
|
84
|
+
isinstance(secret := key.get("secret"), str) and secret): # noqa
|
85
|
+
self._key = {"key": key_id, "secret": secret}
|
86
|
+
self._key_id = key_id
|
87
|
+
self._secret = secret
|
88
|
+
self._key_pair = (key_id, secret)
|
89
|
+
if ((isinstance(server, str) and server) or (isinstance(server := key.get("server"), str) and server)):
|
90
|
+
if server := normalize_server(server):
|
91
|
+
self._key["server"] = self._server = server
|
92
|
+
if not self._key:
|
93
|
+
raise Exception("Portal init error; from key.")
|
94
|
+
|
95
|
+
def init_from_key_pair(key_pair: tuple, server: Optional[str], unspecified: Optional[list] = []) -> None:
|
96
|
+
if len(key_pair) == 2:
|
97
|
+
init_from_key({"key": key_pair[0], "secret": key_pair[1]}, server, unspecified)
|
98
|
+
else:
|
99
|
+
raise Exception("Portal init error; from key-pair.")
|
100
|
+
|
101
|
+
def init_from_keys_file(keys_file: str, env: Optional[str], server: Optional[str],
|
102
|
+
unspecified: Optional[list] = []) -> None:
|
103
|
+
try:
|
104
|
+
with io.open(keys_file) as f:
|
105
|
+
keys = json.load(f)
|
106
|
+
except Exception:
|
107
|
+
raise Exception(f"Portal init error; cannot open keys-file: {keys_file}")
|
108
|
+
if isinstance(env, str) and env and isinstance(key := keys.get(env), dict):
|
109
|
+
init_from_key(key, server)
|
110
|
+
self._keys_file = keys_file
|
111
|
+
self._env = env
|
112
|
+
elif isinstance(server, str) and server and (key := [k for k in keys if keys[k].get("server") == server]):
|
113
|
+
init_from_key(key, server)
|
114
|
+
self._keys_file = keys_file
|
115
|
+
elif len(keys) == 1 and (env := next(iter(keys))) and isinstance(key := keys[env], dict) and key:
|
116
|
+
init_from_key(key, server)
|
117
|
+
self._keys_file = keys_file
|
118
|
+
self._env = env
|
119
|
+
else:
|
120
|
+
raise Exception(f"Portal init error; {env or server or None} not found in keys-file: {keys_file}")
|
121
|
+
|
122
|
+
def init_from_env_server_app(env: str, server: str, app: Optional[str],
|
123
|
+
unspecified: Optional[list] = None) -> None:
|
124
|
+
return init_from_keys_file(self._default_keys_file(app, env), env, server, unspecified)
|
125
|
+
|
126
|
+
def normalize_server(server: str) -> Optional[str]:
|
127
|
+
prefix = ""
|
128
|
+
if (lserver := server.lower()).startswith("http://"):
|
129
|
+
prefix = "http://"
|
130
|
+
elif lserver.startswith("https://"):
|
131
|
+
prefix = "https://"
|
132
|
+
if prefix:
|
133
|
+
if (server := re.sub(r"/+", "/", server[len(prefix):])).startswith("/"):
|
134
|
+
server = server[1:]
|
135
|
+
if len(server) > 1 and server.endswith("/"):
|
136
|
+
server = server[:-1]
|
137
|
+
return prefix + server if server else None
|
138
|
+
|
139
|
+
if isinstance(arg, Portal):
|
140
|
+
init_from_portal(arg, unspecified=[env, server, app])
|
141
|
+
elif isinstance(arg, (TestApp, VirtualApp, Router)):
|
142
|
+
init_from_vapp(arg, unspecified=[env, server, app])
|
143
|
+
elif isinstance(arg, str) and arg.endswith(".ini"):
|
144
|
+
init_from_ini_file(arg, unspecified=[env, server, app])
|
145
|
+
elif isinstance(arg, dict):
|
146
|
+
init_from_key(arg, server, unspecified=[env, app])
|
147
|
+
elif isinstance(arg, tuple):
|
148
|
+
init_from_key_pair(arg, server, unspecified=[env, app])
|
149
|
+
elif isinstance(arg, str) and arg.endswith(".json"):
|
150
|
+
init_from_keys_file(arg, env, server, unspecified=[app])
|
151
|
+
elif isinstance(arg, str) and arg:
|
152
|
+
init_from_env_server_app(arg, server, app, unspecified=[env])
|
153
|
+
elif isinstance(env, str) and env:
|
154
|
+
init_from_env_server_app(env, server, app, unspecified=[arg])
|
155
|
+
else:
|
156
|
+
raise Exception("Portal init error; invalid args.")
|
112
157
|
|
113
158
|
@property
|
114
|
-
def
|
115
|
-
return self.
|
159
|
+
def ini_file(self) -> Optional[str]:
|
160
|
+
return self._ini_file
|
116
161
|
|
117
162
|
@property
|
118
|
-
def key(self):
|
163
|
+
def key(self) -> Optional[dict]:
|
119
164
|
return self._key
|
120
165
|
|
121
166
|
@property
|
122
|
-
def key_pair(self):
|
167
|
+
def key_pair(self) -> Optional[tuple]:
|
123
168
|
return self._key_pair
|
124
169
|
|
125
170
|
@property
|
126
|
-
def key_id(self):
|
171
|
+
def key_id(self) -> Optional[str]:
|
127
172
|
return self._key_id
|
128
173
|
|
129
174
|
@property
|
130
|
-
def
|
131
|
-
return self.
|
175
|
+
def secret(self) -> Optional[str]:
|
176
|
+
return self._secret
|
177
|
+
|
178
|
+
@property
|
179
|
+
def keys_file(self) -> Optional[str]:
|
180
|
+
return self._keys_file
|
181
|
+
|
182
|
+
@property
|
183
|
+
def env(self) -> Optional[str]:
|
184
|
+
return self._env
|
185
|
+
|
186
|
+
@property
|
187
|
+
def server(self) -> Optional[str]:
|
188
|
+
return self._server
|
132
189
|
|
133
190
|
@property
|
134
|
-
def
|
191
|
+
def app(self) -> Optional[str]:
|
192
|
+
return self._app
|
193
|
+
|
194
|
+
@property
|
195
|
+
def vapp(self) -> Optional[TestApp]:
|
135
196
|
return self._vapp
|
136
197
|
|
137
198
|
def get_metadata(self, object_id: str) -> Optional[dict]:
|
@@ -148,27 +209,27 @@ class Portal:
|
|
148
209
|
return self.post(f"/{object_type}", data)
|
149
210
|
|
150
211
|
def get(self, uri: str, follow: bool = True, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
|
151
|
-
if
|
152
|
-
response = self._vapp.get(self.
|
212
|
+
if self._vapp:
|
213
|
+
response = self._vapp.get(self.url(uri), **self._kwargs(**kwargs))
|
153
214
|
if response and response.status_code in [301, 302, 303, 307, 308] and follow:
|
154
215
|
response = response.follow()
|
155
216
|
return self._response(response)
|
156
|
-
return requests.get(self.
|
217
|
+
return requests.get(self.url(uri), allow_redirects=follow, **self._kwargs(**kwargs))
|
157
218
|
|
158
219
|
def patch(self, uri: str, data: Optional[dict] = None,
|
159
220
|
json: Optional[dict] = None, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
|
160
|
-
if
|
161
|
-
return self._vapp.patch_json(self.
|
162
|
-
return requests.patch(self.
|
221
|
+
if self._vapp:
|
222
|
+
return self._vapp.patch_json(self.url(uri), json or data, **self._kwargs(**kwargs))
|
223
|
+
return requests.patch(self.url(uri), data=data, json=json, **self._kwargs(**kwargs))
|
163
224
|
|
164
225
|
def post(self, uri: str, data: Optional[dict] = None, json: Optional[dict] = None,
|
165
226
|
files: Optional[dict] = None, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
|
166
|
-
if
|
227
|
+
if self._vapp:
|
167
228
|
if files:
|
168
|
-
return self._vapp.post(self.
|
229
|
+
return self._vapp.post(self.url(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
|
169
230
|
else:
|
170
|
-
return self._vapp.post_json(self.
|
171
|
-
return requests.post(self.
|
231
|
+
return self._vapp.post_json(self.url(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
|
232
|
+
return requests.post(self.url(uri), data=data, json=json, files=files, **self._kwargs(**kwargs))
|
172
233
|
|
173
234
|
def get_schema(self, schema_name: str) -> Optional[dict]:
|
174
235
|
return get_schema(self.schema_name(schema_name), portal_vapp=self._vapp, key=self._key)
|
@@ -182,7 +243,7 @@ class Portal:
|
|
182
243
|
|
183
244
|
def is_file_schema(self, schema_name: str) -> bool:
|
184
245
|
if super_type_map := self.get_schemas_super_type_map():
|
185
|
-
if file_super_type := super_type_map.get(FILE_SCHEMA_NAME):
|
246
|
+
if file_super_type := super_type_map.get(Portal.FILE_SCHEMA_NAME):
|
186
247
|
return self.schema_name(schema_name) in file_super_type
|
187
248
|
return False
|
188
249
|
|
@@ -222,13 +283,14 @@ class Portal:
|
|
222
283
|
except Exception:
|
223
284
|
return False
|
224
285
|
|
225
|
-
def
|
286
|
+
def url(self, uri: str) -> str:
|
226
287
|
if not isinstance(uri, str) or not uri:
|
227
288
|
return "/"
|
228
|
-
if uri.lower().startswith("http://") or
|
289
|
+
if (luri := uri.lower()).startswith("http://") or luri.startswith("https://"):
|
229
290
|
return uri
|
230
|
-
uri
|
231
|
-
|
291
|
+
if not (uri := re.sub(r"/+", "/", uri)).startswith("/"):
|
292
|
+
uri = "/"
|
293
|
+
return self._server + uri if self._server else uri
|
232
294
|
|
233
295
|
def _kwargs(self, **kwargs) -> dict:
|
234
296
|
result_kwargs = {"headers":
|
@@ -239,6 +301,16 @@ class Portal:
|
|
239
301
|
result_kwargs["timeout"] = timeout
|
240
302
|
return result_kwargs
|
241
303
|
|
304
|
+
def _default_keys_file(self, app: Optional[str], env: Optional[str] = None) -> Optional[str]:
|
305
|
+
def is_valid_app(app: Optional[str]) -> bool: # noqa
|
306
|
+
return app and app.lower() in [name.lower() for name in ORCHESTRATED_APPS]
|
307
|
+
def infer_app_from_env(env: str) -> Optional[str]: # noqa
|
308
|
+
if isinstance(env, str) and (lenv := env.lower()):
|
309
|
+
if app := [app for app in ORCHESTRATED_APPS if lenv.startswith(app.lower())]:
|
310
|
+
return app[0]
|
311
|
+
if is_valid_app(app) or (app := infer_app_from_env(env)):
|
312
|
+
return os.path.join(Portal.KEYS_FILE_DIRECTORY, f".{app.lower()}-keys.json")
|
313
|
+
|
242
314
|
def _response(self, response) -> Optional[RequestResponse]:
|
243
315
|
if response and isinstance(getattr(response.__class__, "json"), property):
|
244
316
|
class RequestResponseWrapper: # For consistency change json property to method.
|
@@ -255,15 +327,15 @@ class Portal:
|
|
255
327
|
@staticmethod
|
256
328
|
def create_for_testing(ini_file: Optional[str] = None) -> Portal:
|
257
329
|
if isinstance(ini_file, str):
|
258
|
-
return Portal(Portal.
|
330
|
+
return Portal(Portal._create_testapp(ini_file))
|
259
331
|
minimal_ini_for_unit_testing = "[app:app]\nuse = egg:encoded\nsqlalchemy.url = postgresql://dummy\n"
|
260
332
|
with temporary_file(content=minimal_ini_for_unit_testing, suffix=".ini") as ini_file:
|
261
|
-
return Portal(Portal.
|
333
|
+
return Portal(Portal._create_testapp(ini_file))
|
262
334
|
|
263
335
|
@staticmethod
|
264
336
|
def create_for_testing_local(ini_file: Optional[str] = None) -> Portal:
|
265
337
|
if isinstance(ini_file, str) and ini_file:
|
266
|
-
return Portal(Portal.
|
338
|
+
return Portal(Portal._create_testapp(ini_file))
|
267
339
|
minimal_ini_for_testing_local = "\n".join([
|
268
340
|
"[app:app]\nuse = egg:encoded\nfile_upload_bucket = dummy",
|
269
341
|
"sqlalchemy.url = postgresql://postgres@localhost:5441/postgres?host=/tmp/snovault/pgdata",
|
@@ -284,11 +356,20 @@ class Portal:
|
|
284
356
|
"multiauth.policy.auth0.base = encoded.authentication.Auth0AuthenticationPolicy"
|
285
357
|
])
|
286
358
|
with temporary_file(content=minimal_ini_for_testing_local, suffix=".ini") as minimal_ini_file:
|
287
|
-
return Portal(Portal.
|
359
|
+
return Portal(Portal._create_testapp(minimal_ini_file))
|
288
360
|
|
289
361
|
@staticmethod
|
290
|
-
def
|
291
|
-
if isinstance(
|
292
|
-
return
|
293
|
-
|
294
|
-
|
362
|
+
def _create_testapp(arg: Union[TestApp, VirtualApp, Router, str] = None, app_name: Optional[str] = None) -> TestApp:
|
363
|
+
if isinstance(arg, TestApp):
|
364
|
+
return arg
|
365
|
+
elif isinstance(arg, VirtualApp):
|
366
|
+
if not isinstance(arg.wrapped_app, TestApp):
|
367
|
+
raise Exception("Portal._create_testapp VirtualApp argument error.")
|
368
|
+
return arg.wrapped_app
|
369
|
+
if isinstance(arg, Router):
|
370
|
+
router = arg
|
371
|
+
elif isinstance(arg, str) or arg is None:
|
372
|
+
router = get_app(arg or "development.ini", app_name or "app")
|
373
|
+
else:
|
374
|
+
raise Exception("Portal._create_testapp argument error.")
|
375
|
+
return TestApp(router, {"HTTP_ACCEPT": "application/json", "REMOTE_USER": "TEST"})
|
dcicutils/structured_data.py
CHANGED
@@ -42,9 +42,9 @@ StructuredDataSet = Type["StructuredDataSet"]
|
|
42
42
|
class StructuredDataSet:
|
43
43
|
|
44
44
|
def __init__(self, file: Optional[str] = None, portal: Optional[Union[VirtualApp, TestApp, Portal]] = None,
|
45
|
-
schemas: Optional[List[dict]] = None,
|
45
|
+
schemas: Optional[List[dict]] = None, autoadd: Optional[dict] = None,
|
46
46
|
order: Optional[List[str]] = None, prune: bool = True) -> None:
|
47
|
-
self.data = {}
|
47
|
+
self.data = {}
|
48
48
|
self._portal = Portal(portal, data=self.data, schemas=schemas) if portal else None
|
49
49
|
self._order = order
|
50
50
|
self._prune = prune
|
@@ -52,13 +52,14 @@ class StructuredDataSet:
|
|
52
52
|
self._errors = {}
|
53
53
|
self._resolved_refs = set()
|
54
54
|
self._validated = False
|
55
|
+
self._autoadd_properties = autoadd if isinstance(autoadd, dict) and autoadd else None
|
55
56
|
self._load_file(file) if file else None
|
56
57
|
|
57
58
|
@staticmethod
|
58
59
|
def load(file: str, portal: Optional[Union[VirtualApp, TestApp, Portal]] = None,
|
59
|
-
schemas: Optional[List[dict]] = None,
|
60
|
+
schemas: Optional[List[dict]] = None, autoadd: Optional[dict] = None,
|
60
61
|
order: Optional[List[str]] = None, prune: bool = True) -> StructuredDataSet:
|
61
|
-
return StructuredDataSet(file=file, portal=portal, schemas=schemas, order=order, prune=prune)
|
62
|
+
return StructuredDataSet(file=file, portal=portal, schemas=schemas, autoadd=autoadd, order=order, prune=prune)
|
62
63
|
|
63
64
|
def validate(self, force: bool = False) -> None:
|
64
65
|
if self._validated and not force:
|
@@ -112,7 +113,7 @@ class StructuredDataSet:
|
|
112
113
|
def _load_file(self, file: str) -> None:
|
113
114
|
# Returns a dictionary where each property is the name (i.e. the type) of the data,
|
114
115
|
# and the value is array of dictionaries for the data itself. Handle these kinds of files:
|
115
|
-
# 1. Single CSV
|
116
|
+
# 1. Single CSV, TSV, or JSON file, where the (base) name of the file is the data type name.
|
116
117
|
# 2. Single Excel file containing one or more sheets, where each sheet
|
117
118
|
# represents (i.e. is named for, and contains data for) a different type.
|
118
119
|
# 3. Zip file (.zip or .tar.gz or .tgz or .tar), containing data files to load, where the
|
@@ -163,6 +164,8 @@ class StructuredDataSet:
|
|
163
164
|
structured_row = structured_row_template.create_row()
|
164
165
|
for column_name, value in row.items():
|
165
166
|
structured_row_template.set_value(structured_row, column_name, value, reader.file, reader.row_number)
|
167
|
+
if self._autoadd_properties:
|
168
|
+
self._add_properties(structured_row, self._autoadd_properties, schema)
|
166
169
|
self._add(type_name, structured_row)
|
167
170
|
self._note_warning(reader.warnings, "reader")
|
168
171
|
if schema:
|
@@ -177,6 +180,11 @@ class StructuredDataSet:
|
|
177
180
|
else:
|
178
181
|
self.data[type_name] = [data] if isinstance(data, dict) else data
|
179
182
|
|
183
|
+
def _add_properties(self, structured_row: dict, properties: dict, schema: Optional[dict] = None) -> None:
|
184
|
+
for name in properties:
|
185
|
+
if name not in structured_row and (not schema or schema.data.get("properties", {}).get(name)):
|
186
|
+
structured_row[name] = properties[name]
|
187
|
+
|
180
188
|
def _note_warning(self, item: Optional[Union[dict, List[dict]]], group: str) -> None:
|
181
189
|
self._note_issue(self._warnings, item, group)
|
182
190
|
|
@@ -475,7 +483,6 @@ class Schema:
|
|
475
483
|
if unique:
|
476
484
|
typeinfo[key]["unique"] = True
|
477
485
|
result.update(typeinfo)
|
478
|
-
# result.update(self._create_typeinfo(array_property_items, parent_key=key))
|
479
486
|
continue
|
480
487
|
result[key] = {"type": property_value_type, "map": self._map_function({**property_value, "column": key})}
|
481
488
|
if ARRAY_NAME_SUFFIX_CHAR in key:
|
@@ -543,16 +550,13 @@ class Portal(PortalBase):
|
|
543
550
|
|
544
551
|
def __init__(self,
|
545
552
|
arg: Optional[Union[VirtualApp, TestApp, Router, Portal, dict, tuple, str]] = None,
|
546
|
-
env: Optional[str] = None,
|
547
|
-
|
548
|
-
|
549
|
-
|
550
|
-
|
551
|
-
|
552
|
-
|
553
|
-
if isinstance(portal, Portal):
|
554
|
-
self._schemas = schemas if schemas is not None else portal._schemas # Explicitly specified/known schemas.
|
555
|
-
self._data = data if data is not None else portal._data # Data set being loaded; e.g. by StructuredDataSet.
|
553
|
+
env: Optional[str] = None, server: Optional[str] = None,
|
554
|
+
app: Optional[OrchestratedApp] = None,
|
555
|
+
data: Optional[dict] = None, schemas: Optional[List[dict]] = None) -> None:
|
556
|
+
super().__init__(arg, env=env, server=server, app=app)
|
557
|
+
if isinstance(arg, Portal):
|
558
|
+
self._schemas = schemas if schemas is not None else arg._schemas # Explicitly specified/known schemas.
|
559
|
+
self._data = data if data is not None else arg._data # Data set being loaded; e.g. by StructuredDataSet.
|
556
560
|
else:
|
557
561
|
self._schemas = schemas
|
558
562
|
self._data = data
|
@@ -43,7 +43,7 @@ dcicutils/log_utils.py,sha256=7pWMc6vyrorUZQf-V-M3YC6zrPgNhuV_fzm9xqTPph0,10883
|
|
43
43
|
dcicutils/misc_utils.py,sha256=nRjLEORY35YmJwTjO0fnauBPznaI_bkVasIW8PccDYM,100179
|
44
44
|
dcicutils/obfuscation_utils.py,sha256=fo2jOmDRC6xWpYX49u80bVNisqRRoPskFNX3ymFAmjw,5963
|
45
45
|
dcicutils/opensearch_utils.py,sha256=V2exmFYW8Xl2_pGFixF4I2Cc549Opwe4PhFi5twC0M8,1017
|
46
|
-
dcicutils/portal_utils.py,sha256=
|
46
|
+
dcicutils/portal_utils.py,sha256=PLIKONo_BhVrf-r5fkXAaxP5IQZRjvPjf83YxvRx0ZE,18393
|
47
47
|
dcicutils/project_utils.py,sha256=qPdCaFmWUVBJw4rw342iUytwdQC0P-XKpK4mhyIulMM,31250
|
48
48
|
dcicutils/qa_checkers.py,sha256=cdXjeL0jCDFDLT8VR8Px78aS10hwNISOO5G_Zv2TZ6M,20534
|
49
49
|
dcicutils/qa_utils.py,sha256=TT0SiJWiuxYvbsIyhK9VO4uV_suxhB6CpuC4qPacCzQ,160208
|
@@ -56,14 +56,14 @@ dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19
|
|
56
56
|
dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
|
57
57
|
dcicutils/snapshot_utils.py,sha256=ymP7PXH6-yEiXAt75w0ldQFciGNqWBClNxC5gfX2FnY,22961
|
58
58
|
dcicutils/ssl_certificate_utils.py,sha256=F0ifz_wnRRN9dfrfsz7aCp4UDLgHEY8LaK7PjnNvrAQ,9707
|
59
|
-
dcicutils/structured_data.py,sha256=
|
59
|
+
dcicutils/structured_data.py,sha256=mWjH6h9ARUvYMHI4ZiwbaoKtoeM0CAOz1UxCHh4HIHE,32874
|
60
60
|
dcicutils/task_utils.py,sha256=MF8ujmTD6-O2AC2gRGPHyGdUrVKgtr8epT5XU8WtNjk,8082
|
61
61
|
dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
|
62
62
|
dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
|
63
63
|
dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
|
64
64
|
dcicutils/zip_utils.py,sha256=0OXR0aLNwyLIZOzIFTM_5DOun7dxIv6TIZbFiithkO0,3276
|
65
|
-
dcicutils-8.5.0.
|
66
|
-
dcicutils-8.5.0.
|
67
|
-
dcicutils-8.5.0.
|
68
|
-
dcicutils-8.5.0.
|
69
|
-
dcicutils-8.5.0.
|
65
|
+
dcicutils-8.5.0.1b6.dist-info/LICENSE.txt,sha256=t0_-jIjqxNnymZoNJe-OltRIuuF8qfhN0ATlHyrUJPk,1102
|
66
|
+
dcicutils-8.5.0.1b6.dist-info/METADATA,sha256=wWiZixknI6Pne33sGS-Lxeh6hSKSScj44tPcctlpHJw,3314
|
67
|
+
dcicutils-8.5.0.1b6.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
|
68
|
+
dcicutils-8.5.0.1b6.dist-info/entry_points.txt,sha256=8wbw5csMIgBXhkwfgsgJeuFcoUc0WsucUxmOyml2aoA,209
|
69
|
+
dcicutils-8.5.0.1b6.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|