dcicutils 8.5.0.1b3__py3-none-any.whl → 8.5.0.1b6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
dcicutils/portal_utils.py CHANGED
@@ -1,19 +1,20 @@
1
1
  from collections import deque
2
+ import io
3
+ import json
2
4
  from pyramid.paster import get_app
3
5
  from pyramid.router import Router
6
+ import os
4
7
  import re
5
8
  import requests
6
9
  from requests.models import Response as RequestResponse
7
10
  from typing import Optional, Type, Union
8
11
  from webtest.app import TestApp, TestResponse
9
- from dcicutils.common import OrchestratedApp, APP_CGAP, APP_FOURFRONT, APP_SMAHT, ORCHESTRATED_APPS
10
- from dcicutils.creds_utils import CGAPKeyManager, FourfrontKeyManager, SMaHTKeyManager
12
+ from dcicutils.common import OrchestratedApp, ORCHESTRATED_APPS
11
13
  from dcicutils.ff_utils import get_metadata, get_schema, patch_metadata, post_metadata
12
14
  from dcicutils.misc_utils import to_camel_case, VirtualApp
13
15
  from dcicutils.zip_utils import temporary_file
14
16
 
15
17
  Portal = Type["Portal"] # Forward type reference for type hints.
16
- FILE_SCHEMA_NAME = "File"
17
18
 
18
19
 
19
20
  class Portal:
@@ -23,115 +24,175 @@ class Portal:
23
24
  2. From a key dictionary, containing "key" and "secret" property values.
24
25
  3. From a key tuple, containing (in order) a key and secret values.
25
26
  4. From a keys file assumed to reside in ~/.{app}-keys.json where the given "app" value is either "smaht", "cgap",
26
- or "fourfront"; and where this file is assumed to contain a dictionary with a key equal to the given "env"
27
- value (e.g. smaht-localhost) and with a dictionary value containing "key" and "secret" property values; if
28
- an "app" value is not specified but the given "env" value begins with one of the app values then that value
29
- will be used, i.e. e.g. if env is "smaht-localhost" and app is unspecified than it is assumed to be "smaht".
27
+ or "fourfront"; where is assumed to contain a dictionary with a key for the given "env" value, e.g. smaht-local;
28
+ and with a dictionary value containing "key" and "secret" property values, and an optional "server" property;
29
+ if an "app" value is not specified but the given "env" value begins with one of the app values then that value
30
+ will be used, i.e. e.g. if "env" is "smaht-local" and app is unspecified than it is assumed to be "smaht".
30
31
  5. From a keys file as described above (#4) but rather than be identified by the given "env" value it
31
- is looked up by the given "server" name and the "server" key dictionary value in the key file.
32
+ is looked up via the given "server" name and the "server" key dictionary value in the key file.
32
33
  6. From a given "vapp" value (which is assumed to be a TestApp or VirtualApp).
33
- 7. From another Portal object.
34
- 8. From a a pyramid Router object.
34
+ 7. From another Portal object; or from a a pyramid Router object.
35
35
  """
36
+ FILE_SCHEMA_NAME = "File"
37
+ KEYS_FILE_DIRECTORY = os.path.expanduser(f"~")
38
+
36
39
  def __init__(self,
37
- arg: Optional[Union[VirtualApp, TestApp, Router, Portal, dict, tuple, str]] = None,
38
- env: Optional[str] = None, app: Optional[OrchestratedApp] = None, server: Optional[str] = None,
39
- key: Optional[Union[dict, tuple]] = None,
40
- vapp: Optional[Union[VirtualApp, TestApp, Router, Portal, str]] = None,
41
- portal: Optional[Union[VirtualApp, TestApp, Router, Portal, str]] = None) -> Portal:
42
- if vapp and not portal:
43
- portal = vapp
44
- if ((isinstance(arg, (VirtualApp, TestApp, Router, Portal)) or
45
- isinstance(arg, str) and arg.endswith(".ini")) and not portal):
46
- portal = arg
47
- elif isinstance(arg, str) and not env:
48
- env = arg
49
- elif (isinstance(arg, dict) or isinstance(arg, tuple)) and not key:
50
- key = arg
51
- if not app and env:
52
- if env.startswith(APP_SMAHT):
53
- app = APP_SMAHT
54
- elif env.startswith(APP_CGAP):
55
- app = APP_CGAP
56
- elif env.startswith(APP_FOURFRONT):
57
- app = APP_FOURFRONT
58
- if isinstance(portal, Portal):
59
- self._vapp = portal._vapp
60
- self._env = portal._env
61
- self._app = portal._app
62
- self._server = portal._server
40
+ arg: Optional[Union[Portal, TestApp, VirtualApp, Router, dict, tuple, str]] = None,
41
+ env: Optional[str] = None, server: Optional[str] = None,
42
+ app: Optional[OrchestratedApp] = None) -> None:
43
+
44
+ def init(unspecified: Optional[list] = []) -> None:
45
+ self._ini_file = None
46
+ self._key = None
47
+ self._key_pair = None
48
+ self._key_id = None
49
+ self._secret = None
50
+ self._keys_file = None
51
+ self._env = None
52
+ self._server = None
53
+ self._app = None
54
+ self._vapp = None
55
+ for arg in unspecified:
56
+ if arg is not None:
57
+ raise Exception("Portal init error; extraneous args.")
58
+
59
+ def init_from_portal(portal: Portal, unspecified: Optional[list] = None) -> None:
60
+ init(unspecified)
61
+ self._ini_file = portal._ini_file
63
62
  self._key = portal._key
64
63
  self._key_pair = portal._key_pair
65
64
  self._key_id = portal._key_id
66
- self._key_file = portal._key_file
67
- return
68
- self._vapp = None
69
- self._env = env
70
- self._app = app
71
- self._server = server
72
- self._key = None
73
- self._key_pair = None
74
- self._key_id = None
75
- self._key_file = None
76
- if isinstance(portal, (VirtualApp, TestApp)):
77
- self._vapp = portal
78
- elif isinstance(portal, (Router, str)):
79
- self._vapp = Portal._create_vapp(portal)
80
- elif isinstance(key, dict):
81
- self._key = key
82
- self._key_pair = (key.get("key"), key.get("secret")) if key else None
83
- if key_server := key.get("server"):
84
- self._server = key_server
85
- elif isinstance(key, tuple) and len(key) >= 2:
86
- self._key = {"key": key[0], "secret": key[1]}
87
- self._key_pair = key
88
- elif isinstance(env, str):
89
- key_managers = {APP_CGAP: CGAPKeyManager, APP_FOURFRONT: FourfrontKeyManager, APP_SMAHT: SMaHTKeyManager}
90
- if not (key_manager := key_managers.get(self._app)) or not (key_manager := key_manager()):
91
- raise Exception(f"Invalid app name: {self._app} (valid: {', '.join(ORCHESTRATED_APPS)}).")
92
- if isinstance(env, str):
93
- self._key = key_manager.get_keydict_for_env(env)
94
- if key_server := self._key.get("server"):
95
- self._server = key_server
96
- elif isinstance(self._server, str):
97
- self._key = key_manager.get_keydict_for_server(self._server)
98
- self._key_pair = key_manager.keydict_to_keypair(self._key) if self._key else None
99
- self._key_file = key_manager.keys_file
100
- if self._key and (key_id := self._key.get("key")):
101
- self._key_id = key_id
102
- elif self._key_pair and (key_id := self._key_pair[1]):
103
- self._key_id = key_id
65
+ self._secret = portal._secret
66
+ self._keys_file = portal._keys_file
67
+ self._env = portal._env
68
+ self._server = portal._server
69
+ self._app = portal._app
70
+ self._vapp = portal._vapp
104
71
 
105
- @property
106
- def env(self):
107
- return self._env
72
+ def init_from_vapp(vapp: Union[TestApp, VirtualApp, Router], unspecified: Optional[list] = []) -> None:
73
+ init(unspecified)
74
+ self._vapp = Portal._create_testapp(vapp)
108
75
 
109
- @property
110
- def app(self):
111
- return self._app
76
+ def init_from_ini_file(ini_file: str, unspecified: Optional[list] = []) -> None:
77
+ init(unspecified)
78
+ self._ini_file = ini_file
79
+ self._vapp = Portal._create_testapp(ini_file)
80
+
81
+ def init_from_key(key: dict, server: Optional[str], unspecified: Optional[list] = []) -> None:
82
+ init(unspecified)
83
+ if (isinstance(key_id := key.get("key"), str) and key_id and
84
+ isinstance(secret := key.get("secret"), str) and secret): # noqa
85
+ self._key = {"key": key_id, "secret": secret}
86
+ self._key_id = key_id
87
+ self._secret = secret
88
+ self._key_pair = (key_id, secret)
89
+ if ((isinstance(server, str) and server) or (isinstance(server := key.get("server"), str) and server)):
90
+ if server := normalize_server(server):
91
+ self._key["server"] = self._server = server
92
+ if not self._key:
93
+ raise Exception("Portal init error; from key.")
94
+
95
+ def init_from_key_pair(key_pair: tuple, server: Optional[str], unspecified: Optional[list] = []) -> None:
96
+ if len(key_pair) == 2:
97
+ init_from_key({"key": key_pair[0], "secret": key_pair[1]}, server, unspecified)
98
+ else:
99
+ raise Exception("Portal init error; from key-pair.")
100
+
101
+ def init_from_keys_file(keys_file: str, env: Optional[str], server: Optional[str],
102
+ unspecified: Optional[list] = []) -> None:
103
+ try:
104
+ with io.open(keys_file) as f:
105
+ keys = json.load(f)
106
+ except Exception:
107
+ raise Exception(f"Portal init error; cannot open keys-file: {keys_file}")
108
+ if isinstance(env, str) and env and isinstance(key := keys.get(env), dict):
109
+ init_from_key(key, server)
110
+ self._keys_file = keys_file
111
+ self._env = env
112
+ elif isinstance(server, str) and server and (key := [k for k in keys if keys[k].get("server") == server]):
113
+ init_from_key(key, server)
114
+ self._keys_file = keys_file
115
+ elif len(keys) == 1 and (env := next(iter(keys))) and isinstance(key := keys[env], dict) and key:
116
+ init_from_key(key, server)
117
+ self._keys_file = keys_file
118
+ self._env = env
119
+ else:
120
+ raise Exception(f"Portal init error; {env or server or None} not found in keys-file: {keys_file}")
121
+
122
+ def init_from_env_server_app(env: str, server: str, app: Optional[str],
123
+ unspecified: Optional[list] = None) -> None:
124
+ return init_from_keys_file(self._default_keys_file(app, env), env, server, unspecified)
125
+
126
+ def normalize_server(server: str) -> Optional[str]:
127
+ prefix = ""
128
+ if (lserver := server.lower()).startswith("http://"):
129
+ prefix = "http://"
130
+ elif lserver.startswith("https://"):
131
+ prefix = "https://"
132
+ if prefix:
133
+ if (server := re.sub(r"/+", "/", server[len(prefix):])).startswith("/"):
134
+ server = server[1:]
135
+ if len(server) > 1 and server.endswith("/"):
136
+ server = server[:-1]
137
+ return prefix + server if server else None
138
+
139
+ if isinstance(arg, Portal):
140
+ init_from_portal(arg, unspecified=[env, server, app])
141
+ elif isinstance(arg, (TestApp, VirtualApp, Router)):
142
+ init_from_vapp(arg, unspecified=[env, server, app])
143
+ elif isinstance(arg, str) and arg.endswith(".ini"):
144
+ init_from_ini_file(arg, unspecified=[env, server, app])
145
+ elif isinstance(arg, dict):
146
+ init_from_key(arg, server, unspecified=[env, app])
147
+ elif isinstance(arg, tuple):
148
+ init_from_key_pair(arg, server, unspecified=[env, app])
149
+ elif isinstance(arg, str) and arg.endswith(".json"):
150
+ init_from_keys_file(arg, env, server, unspecified=[app])
151
+ elif isinstance(arg, str) and arg:
152
+ init_from_env_server_app(arg, server, app, unspecified=[env])
153
+ elif isinstance(env, str) and env:
154
+ init_from_env_server_app(env, server, app, unspecified=[arg])
155
+ else:
156
+ raise Exception("Portal init error; invalid args.")
112
157
 
113
158
  @property
114
- def server(self):
115
- return self._server
159
+ def ini_file(self) -> Optional[str]:
160
+ return self._ini_file
116
161
 
117
162
  @property
118
- def key(self):
163
+ def key(self) -> Optional[dict]:
119
164
  return self._key
120
165
 
121
166
  @property
122
- def key_pair(self):
167
+ def key_pair(self) -> Optional[tuple]:
123
168
  return self._key_pair
124
169
 
125
170
  @property
126
- def key_id(self):
171
+ def key_id(self) -> Optional[str]:
127
172
  return self._key_id
128
173
 
129
174
  @property
130
- def key_file(self):
131
- return self._key_file
175
+ def secret(self) -> Optional[str]:
176
+ return self._secret
177
+
178
+ @property
179
+ def keys_file(self) -> Optional[str]:
180
+ return self._keys_file
181
+
182
+ @property
183
+ def env(self) -> Optional[str]:
184
+ return self._env
185
+
186
+ @property
187
+ def server(self) -> Optional[str]:
188
+ return self._server
132
189
 
133
190
  @property
134
- def vapp(self):
191
+ def app(self) -> Optional[str]:
192
+ return self._app
193
+
194
+ @property
195
+ def vapp(self) -> Optional[TestApp]:
135
196
  return self._vapp
136
197
 
137
198
  def get_metadata(self, object_id: str) -> Optional[dict]:
@@ -148,27 +209,27 @@ class Portal:
148
209
  return self.post(f"/{object_type}", data)
149
210
 
150
211
  def get(self, uri: str, follow: bool = True, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
151
- if isinstance(self._vapp, (VirtualApp, TestApp)):
152
- response = self._vapp.get(self._uri(uri), **self._kwargs(**kwargs))
212
+ if self._vapp:
213
+ response = self._vapp.get(self.url(uri), **self._kwargs(**kwargs))
153
214
  if response and response.status_code in [301, 302, 303, 307, 308] and follow:
154
215
  response = response.follow()
155
216
  return self._response(response)
156
- return requests.get(self._uri(uri), allow_redirects=follow, **self._kwargs(**kwargs))
217
+ return requests.get(self.url(uri), allow_redirects=follow, **self._kwargs(**kwargs))
157
218
 
158
219
  def patch(self, uri: str, data: Optional[dict] = None,
159
220
  json: Optional[dict] = None, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
160
- if isinstance(self._vapp, (VirtualApp, TestApp)):
161
- return self._vapp.patch_json(self._uri(uri), json or data, **self._kwargs(**kwargs))
162
- return requests.patch(self._uri(uri), json=json or data, **self._kwargs(**kwargs))
221
+ if self._vapp:
222
+ return self._vapp.patch_json(self.url(uri), json or data, **self._kwargs(**kwargs))
223
+ return requests.patch(self.url(uri), data=data, json=json, **self._kwargs(**kwargs))
163
224
 
164
225
  def post(self, uri: str, data: Optional[dict] = None, json: Optional[dict] = None,
165
226
  files: Optional[dict] = None, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
166
- if isinstance(self._vapp, (VirtualApp, TestApp)):
227
+ if self._vapp:
167
228
  if files:
168
- return self._vapp.post(self._uri(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
229
+ return self._vapp.post(self.url(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
169
230
  else:
170
- return self._vapp.post_json(self._uri(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
171
- return requests.post(self._uri(uri), json=json or data, files=files, **self._kwargs(**kwargs))
231
+ return self._vapp.post_json(self.url(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
232
+ return requests.post(self.url(uri), data=data, json=json, files=files, **self._kwargs(**kwargs))
172
233
 
173
234
  def get_schema(self, schema_name: str) -> Optional[dict]:
174
235
  return get_schema(self.schema_name(schema_name), portal_vapp=self._vapp, key=self._key)
@@ -182,7 +243,7 @@ class Portal:
182
243
 
183
244
  def is_file_schema(self, schema_name: str) -> bool:
184
245
  if super_type_map := self.get_schemas_super_type_map():
185
- if file_super_type := super_type_map.get(FILE_SCHEMA_NAME):
246
+ if file_super_type := super_type_map.get(Portal.FILE_SCHEMA_NAME):
186
247
  return self.schema_name(schema_name) in file_super_type
187
248
  return False
188
249
 
@@ -222,13 +283,14 @@ class Portal:
222
283
  except Exception:
223
284
  return False
224
285
 
225
- def _uri(self, uri: str) -> str:
286
+ def url(self, uri: str) -> str:
226
287
  if not isinstance(uri, str) or not uri:
227
288
  return "/"
228
- if uri.lower().startswith("http://") or uri.lower().startswith("https://"):
289
+ if (luri := uri.lower()).startswith("http://") or luri.startswith("https://"):
229
290
  return uri
230
- uri = re.sub(r"/+", "/", uri)
231
- return (self._server + ("/" if not uri.startswith("/") else "") + uri) if self._server else uri
291
+ if not (uri := re.sub(r"/+", "/", uri)).startswith("/"):
292
+ uri = "/"
293
+ return self._server + uri if self._server else uri
232
294
 
233
295
  def _kwargs(self, **kwargs) -> dict:
234
296
  result_kwargs = {"headers":
@@ -239,6 +301,16 @@ class Portal:
239
301
  result_kwargs["timeout"] = timeout
240
302
  return result_kwargs
241
303
 
304
+ def _default_keys_file(self, app: Optional[str], env: Optional[str] = None) -> Optional[str]:
305
+ def is_valid_app(app: Optional[str]) -> bool: # noqa
306
+ return app and app.lower() in [name.lower() for name in ORCHESTRATED_APPS]
307
+ def infer_app_from_env(env: str) -> Optional[str]: # noqa
308
+ if isinstance(env, str) and (lenv := env.lower()):
309
+ if app := [app for app in ORCHESTRATED_APPS if lenv.startswith(app.lower())]:
310
+ return app[0]
311
+ if is_valid_app(app) or (app := infer_app_from_env(env)):
312
+ return os.path.join(Portal.KEYS_FILE_DIRECTORY, f".{app.lower()}-keys.json")
313
+
242
314
  def _response(self, response) -> Optional[RequestResponse]:
243
315
  if response and isinstance(getattr(response.__class__, "json"), property):
244
316
  class RequestResponseWrapper: # For consistency change json property to method.
@@ -255,15 +327,15 @@ class Portal:
255
327
  @staticmethod
256
328
  def create_for_testing(ini_file: Optional[str] = None) -> Portal:
257
329
  if isinstance(ini_file, str):
258
- return Portal(Portal._create_vapp(ini_file))
330
+ return Portal(Portal._create_testapp(ini_file))
259
331
  minimal_ini_for_unit_testing = "[app:app]\nuse = egg:encoded\nsqlalchemy.url = postgresql://dummy\n"
260
332
  with temporary_file(content=minimal_ini_for_unit_testing, suffix=".ini") as ini_file:
261
- return Portal(Portal._create_vapp(ini_file))
333
+ return Portal(Portal._create_testapp(ini_file))
262
334
 
263
335
  @staticmethod
264
336
  def create_for_testing_local(ini_file: Optional[str] = None) -> Portal:
265
337
  if isinstance(ini_file, str) and ini_file:
266
- return Portal(Portal._create_vapp(ini_file))
338
+ return Portal(Portal._create_testapp(ini_file))
267
339
  minimal_ini_for_testing_local = "\n".join([
268
340
  "[app:app]\nuse = egg:encoded\nfile_upload_bucket = dummy",
269
341
  "sqlalchemy.url = postgresql://postgres@localhost:5441/postgres?host=/tmp/snovault/pgdata",
@@ -284,11 +356,20 @@ class Portal:
284
356
  "multiauth.policy.auth0.base = encoded.authentication.Auth0AuthenticationPolicy"
285
357
  ])
286
358
  with temporary_file(content=minimal_ini_for_testing_local, suffix=".ini") as minimal_ini_file:
287
- return Portal(Portal._create_vapp(minimal_ini_file))
359
+ return Portal(Portal._create_testapp(minimal_ini_file))
288
360
 
289
361
  @staticmethod
290
- def _create_vapp(value: Union[str, Router, TestApp] = "development.ini", app_name: str = "app") -> TestApp:
291
- if isinstance(value, TestApp):
292
- return value
293
- app = value if isinstance(value, Router) else get_app(value, app_name)
294
- return TestApp(app, {"HTTP_ACCEPT": "application/json", "REMOTE_USER": "TEST"})
362
+ def _create_testapp(arg: Union[TestApp, VirtualApp, Router, str] = None, app_name: Optional[str] = None) -> TestApp:
363
+ if isinstance(arg, TestApp):
364
+ return arg
365
+ elif isinstance(arg, VirtualApp):
366
+ if not isinstance(arg.wrapped_app, TestApp):
367
+ raise Exception("Portal._create_testapp VirtualApp argument error.")
368
+ return arg.wrapped_app
369
+ if isinstance(arg, Router):
370
+ router = arg
371
+ elif isinstance(arg, str) or arg is None:
372
+ router = get_app(arg or "development.ini", app_name or "app")
373
+ else:
374
+ raise Exception("Portal._create_testapp argument error.")
375
+ return TestApp(router, {"HTTP_ACCEPT": "application/json", "REMOTE_USER": "TEST"})
@@ -42,9 +42,9 @@ StructuredDataSet = Type["StructuredDataSet"]
42
42
  class StructuredDataSet:
43
43
 
44
44
  def __init__(self, file: Optional[str] = None, portal: Optional[Union[VirtualApp, TestApp, Portal]] = None,
45
- schemas: Optional[List[dict]] = None, data: Optional[List[dict]] = None,
45
+ schemas: Optional[List[dict]] = None, autoadd: Optional[dict] = None,
46
46
  order: Optional[List[str]] = None, prune: bool = True) -> None:
47
- self.data = {} if not data else data # If portal is None then no schemas nor refs.
47
+ self.data = {}
48
48
  self._portal = Portal(portal, data=self.data, schemas=schemas) if portal else None
49
49
  self._order = order
50
50
  self._prune = prune
@@ -52,13 +52,14 @@ class StructuredDataSet:
52
52
  self._errors = {}
53
53
  self._resolved_refs = set()
54
54
  self._validated = False
55
+ self._autoadd_properties = autoadd if isinstance(autoadd, dict) and autoadd else None
55
56
  self._load_file(file) if file else None
56
57
 
57
58
  @staticmethod
58
59
  def load(file: str, portal: Optional[Union[VirtualApp, TestApp, Portal]] = None,
59
- schemas: Optional[List[dict]] = None,
60
+ schemas: Optional[List[dict]] = None, autoadd: Optional[dict] = None,
60
61
  order: Optional[List[str]] = None, prune: bool = True) -> StructuredDataSet:
61
- return StructuredDataSet(file=file, portal=portal, schemas=schemas, order=order, prune=prune)
62
+ return StructuredDataSet(file=file, portal=portal, schemas=schemas, autoadd=autoadd, order=order, prune=prune)
62
63
 
63
64
  def validate(self, force: bool = False) -> None:
64
65
  if self._validated and not force:
@@ -112,7 +113,7 @@ class StructuredDataSet:
112
113
  def _load_file(self, file: str) -> None:
113
114
  # Returns a dictionary where each property is the name (i.e. the type) of the data,
114
115
  # and the value is array of dictionaries for the data itself. Handle these kinds of files:
115
- # 1. Single CSV of JSON file, where the (base) name of the file is the data type name.
116
+ # 1. Single CSV, TSV, or JSON file, where the (base) name of the file is the data type name.
116
117
  # 2. Single Excel file containing one or more sheets, where each sheet
117
118
  # represents (i.e. is named for, and contains data for) a different type.
118
119
  # 3. Zip file (.zip or .tar.gz or .tgz or .tar), containing data files to load, where the
@@ -163,6 +164,8 @@ class StructuredDataSet:
163
164
  structured_row = structured_row_template.create_row()
164
165
  for column_name, value in row.items():
165
166
  structured_row_template.set_value(structured_row, column_name, value, reader.file, reader.row_number)
167
+ if self._autoadd_properties:
168
+ self._add_properties(structured_row, self._autoadd_properties, schema)
166
169
  self._add(type_name, structured_row)
167
170
  self._note_warning(reader.warnings, "reader")
168
171
  if schema:
@@ -177,6 +180,11 @@ class StructuredDataSet:
177
180
  else:
178
181
  self.data[type_name] = [data] if isinstance(data, dict) else data
179
182
 
183
+ def _add_properties(self, structured_row: dict, properties: dict, schema: Optional[dict] = None) -> None:
184
+ for name in properties:
185
+ if name not in structured_row and (not schema or schema.data.get("properties", {}).get(name)):
186
+ structured_row[name] = properties[name]
187
+
180
188
  def _note_warning(self, item: Optional[Union[dict, List[dict]]], group: str) -> None:
181
189
  self._note_issue(self._warnings, item, group)
182
190
 
@@ -475,7 +483,6 @@ class Schema:
475
483
  if unique:
476
484
  typeinfo[key]["unique"] = True
477
485
  result.update(typeinfo)
478
- # result.update(self._create_typeinfo(array_property_items, parent_key=key))
479
486
  continue
480
487
  result[key] = {"type": property_value_type, "map": self._map_function({**property_value, "column": key})}
481
488
  if ARRAY_NAME_SUFFIX_CHAR in key:
@@ -543,16 +550,13 @@ class Portal(PortalBase):
543
550
 
544
551
  def __init__(self,
545
552
  arg: Optional[Union[VirtualApp, TestApp, Router, Portal, dict, tuple, str]] = None,
546
- env: Optional[str] = None, app: OrchestratedApp = None, server: Optional[str] = None,
547
- key: Optional[Union[dict, tuple]] = None,
548
- portal: Optional[Union[VirtualApp, TestApp, Router, Portal, str]] = None,
549
- data: Optional[dict] = None, schemas: Optional[List[dict]] = None) -> Optional[Portal]:
550
- super().__init__(arg, env=env, app=app, server=server, key=key, portal=portal)
551
- if isinstance(arg, Portal) and not portal:
552
- portal = arg
553
- if isinstance(portal, Portal):
554
- self._schemas = schemas if schemas is not None else portal._schemas # Explicitly specified/known schemas.
555
- self._data = data if data is not None else portal._data # Data set being loaded; e.g. by StructuredDataSet.
553
+ env: Optional[str] = None, server: Optional[str] = None,
554
+ app: Optional[OrchestratedApp] = None,
555
+ data: Optional[dict] = None, schemas: Optional[List[dict]] = None) -> None:
556
+ super().__init__(arg, env=env, server=server, app=app)
557
+ if isinstance(arg, Portal):
558
+ self._schemas = schemas if schemas is not None else arg._schemas # Explicitly specified/known schemas.
559
+ self._data = data if data is not None else arg._data # Data set being loaded; e.g. by StructuredDataSet.
556
560
  else:
557
561
  self._schemas = schemas
558
562
  self._data = data
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dcicutils
3
- Version: 8.5.0.1b3
3
+ Version: 8.5.0.1b6
4
4
  Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
5
5
  Home-page: https://github.com/4dn-dcic/utils
6
6
  License: MIT
@@ -43,7 +43,7 @@ dcicutils/log_utils.py,sha256=7pWMc6vyrorUZQf-V-M3YC6zrPgNhuV_fzm9xqTPph0,10883
43
43
  dcicutils/misc_utils.py,sha256=nRjLEORY35YmJwTjO0fnauBPznaI_bkVasIW8PccDYM,100179
44
44
  dcicutils/obfuscation_utils.py,sha256=fo2jOmDRC6xWpYX49u80bVNisqRRoPskFNX3ymFAmjw,5963
45
45
  dcicutils/opensearch_utils.py,sha256=V2exmFYW8Xl2_pGFixF4I2Cc549Opwe4PhFi5twC0M8,1017
46
- dcicutils/portal_utils.py,sha256=Jp33fgq3wKKYVMQw851EUKgayrQL9_xsm8NKyAP0Vy4,14553
46
+ dcicutils/portal_utils.py,sha256=PLIKONo_BhVrf-r5fkXAaxP5IQZRjvPjf83YxvRx0ZE,18393
47
47
  dcicutils/project_utils.py,sha256=qPdCaFmWUVBJw4rw342iUytwdQC0P-XKpK4mhyIulMM,31250
48
48
  dcicutils/qa_checkers.py,sha256=cdXjeL0jCDFDLT8VR8Px78aS10hwNISOO5G_Zv2TZ6M,20534
49
49
  dcicutils/qa_utils.py,sha256=TT0SiJWiuxYvbsIyhK9VO4uV_suxhB6CpuC4qPacCzQ,160208
@@ -56,14 +56,14 @@ dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19
56
56
  dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
57
57
  dcicutils/snapshot_utils.py,sha256=ymP7PXH6-yEiXAt75w0ldQFciGNqWBClNxC5gfX2FnY,22961
58
58
  dcicutils/ssl_certificate_utils.py,sha256=F0ifz_wnRRN9dfrfsz7aCp4UDLgHEY8LaK7PjnNvrAQ,9707
59
- dcicutils/structured_data.py,sha256=8jEPKeoVOcqhG_i-H6cK4zBXTUFpFi_9ykw5g1h0n4g,32690
59
+ dcicutils/structured_data.py,sha256=mWjH6h9ARUvYMHI4ZiwbaoKtoeM0CAOz1UxCHh4HIHE,32874
60
60
  dcicutils/task_utils.py,sha256=MF8ujmTD6-O2AC2gRGPHyGdUrVKgtr8epT5XU8WtNjk,8082
61
61
  dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
62
62
  dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
63
63
  dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
64
64
  dcicutils/zip_utils.py,sha256=0OXR0aLNwyLIZOzIFTM_5DOun7dxIv6TIZbFiithkO0,3276
65
- dcicutils-8.5.0.1b3.dist-info/LICENSE.txt,sha256=t0_-jIjqxNnymZoNJe-OltRIuuF8qfhN0ATlHyrUJPk,1102
66
- dcicutils-8.5.0.1b3.dist-info/METADATA,sha256=xLOqwEx3paC3otYf4koYSgOZ8WekQIkBVaKZrF-tH84,3314
67
- dcicutils-8.5.0.1b3.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
68
- dcicutils-8.5.0.1b3.dist-info/entry_points.txt,sha256=8wbw5csMIgBXhkwfgsgJeuFcoUc0WsucUxmOyml2aoA,209
69
- dcicutils-8.5.0.1b3.dist-info/RECORD,,
65
+ dcicutils-8.5.0.1b6.dist-info/LICENSE.txt,sha256=t0_-jIjqxNnymZoNJe-OltRIuuF8qfhN0ATlHyrUJPk,1102
66
+ dcicutils-8.5.0.1b6.dist-info/METADATA,sha256=wWiZixknI6Pne33sGS-Lxeh6hSKSScj44tPcctlpHJw,3314
67
+ dcicutils-8.5.0.1b6.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
68
+ dcicutils-8.5.0.1b6.dist-info/entry_points.txt,sha256=8wbw5csMIgBXhkwfgsgJeuFcoUc0WsucUxmOyml2aoA,209
69
+ dcicutils-8.5.0.1b6.dist-info/RECORD,,