dcicutils 8.5.0.1b3__py3-none-any.whl → 8.5.0.1b6__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
dcicutils/portal_utils.py CHANGED
@@ -1,19 +1,20 @@
1
1
  from collections import deque
2
+ import io
3
+ import json
2
4
  from pyramid.paster import get_app
3
5
  from pyramid.router import Router
6
+ import os
4
7
  import re
5
8
  import requests
6
9
  from requests.models import Response as RequestResponse
7
10
  from typing import Optional, Type, Union
8
11
  from webtest.app import TestApp, TestResponse
9
- from dcicutils.common import OrchestratedApp, APP_CGAP, APP_FOURFRONT, APP_SMAHT, ORCHESTRATED_APPS
10
- from dcicutils.creds_utils import CGAPKeyManager, FourfrontKeyManager, SMaHTKeyManager
12
+ from dcicutils.common import OrchestratedApp, ORCHESTRATED_APPS
11
13
  from dcicutils.ff_utils import get_metadata, get_schema, patch_metadata, post_metadata
12
14
  from dcicutils.misc_utils import to_camel_case, VirtualApp
13
15
  from dcicutils.zip_utils import temporary_file
14
16
 
15
17
  Portal = Type["Portal"] # Forward type reference for type hints.
16
- FILE_SCHEMA_NAME = "File"
17
18
 
18
19
 
19
20
  class Portal:
@@ -23,115 +24,175 @@ class Portal:
23
24
  2. From a key dictionary, containing "key" and "secret" property values.
24
25
  3. From a key tuple, containing (in order) a key and secret values.
25
26
  4. From a keys file assumed to reside in ~/.{app}-keys.json where the given "app" value is either "smaht", "cgap",
26
- or "fourfront"; and where this file is assumed to contain a dictionary with a key equal to the given "env"
27
- value (e.g. smaht-localhost) and with a dictionary value containing "key" and "secret" property values; if
28
- an "app" value is not specified but the given "env" value begins with one of the app values then that value
29
- will be used, i.e. e.g. if env is "smaht-localhost" and app is unspecified than it is assumed to be "smaht".
27
+ or "fourfront"; where is assumed to contain a dictionary with a key for the given "env" value, e.g. smaht-local;
28
+ and with a dictionary value containing "key" and "secret" property values, and an optional "server" property;
29
+ if an "app" value is not specified but the given "env" value begins with one of the app values then that value
30
+ will be used, i.e. e.g. if "env" is "smaht-local" and app is unspecified than it is assumed to be "smaht".
30
31
  5. From a keys file as described above (#4) but rather than be identified by the given "env" value it
31
- is looked up by the given "server" name and the "server" key dictionary value in the key file.
32
+ is looked up via the given "server" name and the "server" key dictionary value in the key file.
32
33
  6. From a given "vapp" value (which is assumed to be a TestApp or VirtualApp).
33
- 7. From another Portal object.
34
- 8. From a a pyramid Router object.
34
+ 7. From another Portal object; or from a a pyramid Router object.
35
35
  """
36
+ FILE_SCHEMA_NAME = "File"
37
+ KEYS_FILE_DIRECTORY = os.path.expanduser(f"~")
38
+
36
39
  def __init__(self,
37
- arg: Optional[Union[VirtualApp, TestApp, Router, Portal, dict, tuple, str]] = None,
38
- env: Optional[str] = None, app: Optional[OrchestratedApp] = None, server: Optional[str] = None,
39
- key: Optional[Union[dict, tuple]] = None,
40
- vapp: Optional[Union[VirtualApp, TestApp, Router, Portal, str]] = None,
41
- portal: Optional[Union[VirtualApp, TestApp, Router, Portal, str]] = None) -> Portal:
42
- if vapp and not portal:
43
- portal = vapp
44
- if ((isinstance(arg, (VirtualApp, TestApp, Router, Portal)) or
45
- isinstance(arg, str) and arg.endswith(".ini")) and not portal):
46
- portal = arg
47
- elif isinstance(arg, str) and not env:
48
- env = arg
49
- elif (isinstance(arg, dict) or isinstance(arg, tuple)) and not key:
50
- key = arg
51
- if not app and env:
52
- if env.startswith(APP_SMAHT):
53
- app = APP_SMAHT
54
- elif env.startswith(APP_CGAP):
55
- app = APP_CGAP
56
- elif env.startswith(APP_FOURFRONT):
57
- app = APP_FOURFRONT
58
- if isinstance(portal, Portal):
59
- self._vapp = portal._vapp
60
- self._env = portal._env
61
- self._app = portal._app
62
- self._server = portal._server
40
+ arg: Optional[Union[Portal, TestApp, VirtualApp, Router, dict, tuple, str]] = None,
41
+ env: Optional[str] = None, server: Optional[str] = None,
42
+ app: Optional[OrchestratedApp] = None) -> None:
43
+
44
+ def init(unspecified: Optional[list] = []) -> None:
45
+ self._ini_file = None
46
+ self._key = None
47
+ self._key_pair = None
48
+ self._key_id = None
49
+ self._secret = None
50
+ self._keys_file = None
51
+ self._env = None
52
+ self._server = None
53
+ self._app = None
54
+ self._vapp = None
55
+ for arg in unspecified:
56
+ if arg is not None:
57
+ raise Exception("Portal init error; extraneous args.")
58
+
59
+ def init_from_portal(portal: Portal, unspecified: Optional[list] = None) -> None:
60
+ init(unspecified)
61
+ self._ini_file = portal._ini_file
63
62
  self._key = portal._key
64
63
  self._key_pair = portal._key_pair
65
64
  self._key_id = portal._key_id
66
- self._key_file = portal._key_file
67
- return
68
- self._vapp = None
69
- self._env = env
70
- self._app = app
71
- self._server = server
72
- self._key = None
73
- self._key_pair = None
74
- self._key_id = None
75
- self._key_file = None
76
- if isinstance(portal, (VirtualApp, TestApp)):
77
- self._vapp = portal
78
- elif isinstance(portal, (Router, str)):
79
- self._vapp = Portal._create_vapp(portal)
80
- elif isinstance(key, dict):
81
- self._key = key
82
- self._key_pair = (key.get("key"), key.get("secret")) if key else None
83
- if key_server := key.get("server"):
84
- self._server = key_server
85
- elif isinstance(key, tuple) and len(key) >= 2:
86
- self._key = {"key": key[0], "secret": key[1]}
87
- self._key_pair = key
88
- elif isinstance(env, str):
89
- key_managers = {APP_CGAP: CGAPKeyManager, APP_FOURFRONT: FourfrontKeyManager, APP_SMAHT: SMaHTKeyManager}
90
- if not (key_manager := key_managers.get(self._app)) or not (key_manager := key_manager()):
91
- raise Exception(f"Invalid app name: {self._app} (valid: {', '.join(ORCHESTRATED_APPS)}).")
92
- if isinstance(env, str):
93
- self._key = key_manager.get_keydict_for_env(env)
94
- if key_server := self._key.get("server"):
95
- self._server = key_server
96
- elif isinstance(self._server, str):
97
- self._key = key_manager.get_keydict_for_server(self._server)
98
- self._key_pair = key_manager.keydict_to_keypair(self._key) if self._key else None
99
- self._key_file = key_manager.keys_file
100
- if self._key and (key_id := self._key.get("key")):
101
- self._key_id = key_id
102
- elif self._key_pair and (key_id := self._key_pair[1]):
103
- self._key_id = key_id
65
+ self._secret = portal._secret
66
+ self._keys_file = portal._keys_file
67
+ self._env = portal._env
68
+ self._server = portal._server
69
+ self._app = portal._app
70
+ self._vapp = portal._vapp
104
71
 
105
- @property
106
- def env(self):
107
- return self._env
72
+ def init_from_vapp(vapp: Union[TestApp, VirtualApp, Router], unspecified: Optional[list] = []) -> None:
73
+ init(unspecified)
74
+ self._vapp = Portal._create_testapp(vapp)
108
75
 
109
- @property
110
- def app(self):
111
- return self._app
76
+ def init_from_ini_file(ini_file: str, unspecified: Optional[list] = []) -> None:
77
+ init(unspecified)
78
+ self._ini_file = ini_file
79
+ self._vapp = Portal._create_testapp(ini_file)
80
+
81
+ def init_from_key(key: dict, server: Optional[str], unspecified: Optional[list] = []) -> None:
82
+ init(unspecified)
83
+ if (isinstance(key_id := key.get("key"), str) and key_id and
84
+ isinstance(secret := key.get("secret"), str) and secret): # noqa
85
+ self._key = {"key": key_id, "secret": secret}
86
+ self._key_id = key_id
87
+ self._secret = secret
88
+ self._key_pair = (key_id, secret)
89
+ if ((isinstance(server, str) and server) or (isinstance(server := key.get("server"), str) and server)):
90
+ if server := normalize_server(server):
91
+ self._key["server"] = self._server = server
92
+ if not self._key:
93
+ raise Exception("Portal init error; from key.")
94
+
95
+ def init_from_key_pair(key_pair: tuple, server: Optional[str], unspecified: Optional[list] = []) -> None:
96
+ if len(key_pair) == 2:
97
+ init_from_key({"key": key_pair[0], "secret": key_pair[1]}, server, unspecified)
98
+ else:
99
+ raise Exception("Portal init error; from key-pair.")
100
+
101
+ def init_from_keys_file(keys_file: str, env: Optional[str], server: Optional[str],
102
+ unspecified: Optional[list] = []) -> None:
103
+ try:
104
+ with io.open(keys_file) as f:
105
+ keys = json.load(f)
106
+ except Exception:
107
+ raise Exception(f"Portal init error; cannot open keys-file: {keys_file}")
108
+ if isinstance(env, str) and env and isinstance(key := keys.get(env), dict):
109
+ init_from_key(key, server)
110
+ self._keys_file = keys_file
111
+ self._env = env
112
+ elif isinstance(server, str) and server and (key := [k for k in keys if keys[k].get("server") == server]):
113
+ init_from_key(key, server)
114
+ self._keys_file = keys_file
115
+ elif len(keys) == 1 and (env := next(iter(keys))) and isinstance(key := keys[env], dict) and key:
116
+ init_from_key(key, server)
117
+ self._keys_file = keys_file
118
+ self._env = env
119
+ else:
120
+ raise Exception(f"Portal init error; {env or server or None} not found in keys-file: {keys_file}")
121
+
122
+ def init_from_env_server_app(env: str, server: str, app: Optional[str],
123
+ unspecified: Optional[list] = None) -> None:
124
+ return init_from_keys_file(self._default_keys_file(app, env), env, server, unspecified)
125
+
126
+ def normalize_server(server: str) -> Optional[str]:
127
+ prefix = ""
128
+ if (lserver := server.lower()).startswith("http://"):
129
+ prefix = "http://"
130
+ elif lserver.startswith("https://"):
131
+ prefix = "https://"
132
+ if prefix:
133
+ if (server := re.sub(r"/+", "/", server[len(prefix):])).startswith("/"):
134
+ server = server[1:]
135
+ if len(server) > 1 and server.endswith("/"):
136
+ server = server[:-1]
137
+ return prefix + server if server else None
138
+
139
+ if isinstance(arg, Portal):
140
+ init_from_portal(arg, unspecified=[env, server, app])
141
+ elif isinstance(arg, (TestApp, VirtualApp, Router)):
142
+ init_from_vapp(arg, unspecified=[env, server, app])
143
+ elif isinstance(arg, str) and arg.endswith(".ini"):
144
+ init_from_ini_file(arg, unspecified=[env, server, app])
145
+ elif isinstance(arg, dict):
146
+ init_from_key(arg, server, unspecified=[env, app])
147
+ elif isinstance(arg, tuple):
148
+ init_from_key_pair(arg, server, unspecified=[env, app])
149
+ elif isinstance(arg, str) and arg.endswith(".json"):
150
+ init_from_keys_file(arg, env, server, unspecified=[app])
151
+ elif isinstance(arg, str) and arg:
152
+ init_from_env_server_app(arg, server, app, unspecified=[env])
153
+ elif isinstance(env, str) and env:
154
+ init_from_env_server_app(env, server, app, unspecified=[arg])
155
+ else:
156
+ raise Exception("Portal init error; invalid args.")
112
157
 
113
158
  @property
114
- def server(self):
115
- return self._server
159
+ def ini_file(self) -> Optional[str]:
160
+ return self._ini_file
116
161
 
117
162
  @property
118
- def key(self):
163
+ def key(self) -> Optional[dict]:
119
164
  return self._key
120
165
 
121
166
  @property
122
- def key_pair(self):
167
+ def key_pair(self) -> Optional[tuple]:
123
168
  return self._key_pair
124
169
 
125
170
  @property
126
- def key_id(self):
171
+ def key_id(self) -> Optional[str]:
127
172
  return self._key_id
128
173
 
129
174
  @property
130
- def key_file(self):
131
- return self._key_file
175
+ def secret(self) -> Optional[str]:
176
+ return self._secret
177
+
178
+ @property
179
+ def keys_file(self) -> Optional[str]:
180
+ return self._keys_file
181
+
182
+ @property
183
+ def env(self) -> Optional[str]:
184
+ return self._env
185
+
186
+ @property
187
+ def server(self) -> Optional[str]:
188
+ return self._server
132
189
 
133
190
  @property
134
- def vapp(self):
191
+ def app(self) -> Optional[str]:
192
+ return self._app
193
+
194
+ @property
195
+ def vapp(self) -> Optional[TestApp]:
135
196
  return self._vapp
136
197
 
137
198
  def get_metadata(self, object_id: str) -> Optional[dict]:
@@ -148,27 +209,27 @@ class Portal:
148
209
  return self.post(f"/{object_type}", data)
149
210
 
150
211
  def get(self, uri: str, follow: bool = True, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
151
- if isinstance(self._vapp, (VirtualApp, TestApp)):
152
- response = self._vapp.get(self._uri(uri), **self._kwargs(**kwargs))
212
+ if self._vapp:
213
+ response = self._vapp.get(self.url(uri), **self._kwargs(**kwargs))
153
214
  if response and response.status_code in [301, 302, 303, 307, 308] and follow:
154
215
  response = response.follow()
155
216
  return self._response(response)
156
- return requests.get(self._uri(uri), allow_redirects=follow, **self._kwargs(**kwargs))
217
+ return requests.get(self.url(uri), allow_redirects=follow, **self._kwargs(**kwargs))
157
218
 
158
219
  def patch(self, uri: str, data: Optional[dict] = None,
159
220
  json: Optional[dict] = None, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
160
- if isinstance(self._vapp, (VirtualApp, TestApp)):
161
- return self._vapp.patch_json(self._uri(uri), json or data, **self._kwargs(**kwargs))
162
- return requests.patch(self._uri(uri), json=json or data, **self._kwargs(**kwargs))
221
+ if self._vapp:
222
+ return self._vapp.patch_json(self.url(uri), json or data, **self._kwargs(**kwargs))
223
+ return requests.patch(self.url(uri), data=data, json=json, **self._kwargs(**kwargs))
163
224
 
164
225
  def post(self, uri: str, data: Optional[dict] = None, json: Optional[dict] = None,
165
226
  files: Optional[dict] = None, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
166
- if isinstance(self._vapp, (VirtualApp, TestApp)):
227
+ if self._vapp:
167
228
  if files:
168
- return self._vapp.post(self._uri(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
229
+ return self._vapp.post(self.url(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
169
230
  else:
170
- return self._vapp.post_json(self._uri(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
171
- return requests.post(self._uri(uri), json=json or data, files=files, **self._kwargs(**kwargs))
231
+ return self._vapp.post_json(self.url(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
232
+ return requests.post(self.url(uri), data=data, json=json, files=files, **self._kwargs(**kwargs))
172
233
 
173
234
  def get_schema(self, schema_name: str) -> Optional[dict]:
174
235
  return get_schema(self.schema_name(schema_name), portal_vapp=self._vapp, key=self._key)
@@ -182,7 +243,7 @@ class Portal:
182
243
 
183
244
  def is_file_schema(self, schema_name: str) -> bool:
184
245
  if super_type_map := self.get_schemas_super_type_map():
185
- if file_super_type := super_type_map.get(FILE_SCHEMA_NAME):
246
+ if file_super_type := super_type_map.get(Portal.FILE_SCHEMA_NAME):
186
247
  return self.schema_name(schema_name) in file_super_type
187
248
  return False
188
249
 
@@ -222,13 +283,14 @@ class Portal:
222
283
  except Exception:
223
284
  return False
224
285
 
225
- def _uri(self, uri: str) -> str:
286
+ def url(self, uri: str) -> str:
226
287
  if not isinstance(uri, str) or not uri:
227
288
  return "/"
228
- if uri.lower().startswith("http://") or uri.lower().startswith("https://"):
289
+ if (luri := uri.lower()).startswith("http://") or luri.startswith("https://"):
229
290
  return uri
230
- uri = re.sub(r"/+", "/", uri)
231
- return (self._server + ("/" if not uri.startswith("/") else "") + uri) if self._server else uri
291
+ if not (uri := re.sub(r"/+", "/", uri)).startswith("/"):
292
+ uri = "/"
293
+ return self._server + uri if self._server else uri
232
294
 
233
295
  def _kwargs(self, **kwargs) -> dict:
234
296
  result_kwargs = {"headers":
@@ -239,6 +301,16 @@ class Portal:
239
301
  result_kwargs["timeout"] = timeout
240
302
  return result_kwargs
241
303
 
304
+ def _default_keys_file(self, app: Optional[str], env: Optional[str] = None) -> Optional[str]:
305
+ def is_valid_app(app: Optional[str]) -> bool: # noqa
306
+ return app and app.lower() in [name.lower() for name in ORCHESTRATED_APPS]
307
+ def infer_app_from_env(env: str) -> Optional[str]: # noqa
308
+ if isinstance(env, str) and (lenv := env.lower()):
309
+ if app := [app for app in ORCHESTRATED_APPS if lenv.startswith(app.lower())]:
310
+ return app[0]
311
+ if is_valid_app(app) or (app := infer_app_from_env(env)):
312
+ return os.path.join(Portal.KEYS_FILE_DIRECTORY, f".{app.lower()}-keys.json")
313
+
242
314
  def _response(self, response) -> Optional[RequestResponse]:
243
315
  if response and isinstance(getattr(response.__class__, "json"), property):
244
316
  class RequestResponseWrapper: # For consistency change json property to method.
@@ -255,15 +327,15 @@ class Portal:
255
327
  @staticmethod
256
328
  def create_for_testing(ini_file: Optional[str] = None) -> Portal:
257
329
  if isinstance(ini_file, str):
258
- return Portal(Portal._create_vapp(ini_file))
330
+ return Portal(Portal._create_testapp(ini_file))
259
331
  minimal_ini_for_unit_testing = "[app:app]\nuse = egg:encoded\nsqlalchemy.url = postgresql://dummy\n"
260
332
  with temporary_file(content=minimal_ini_for_unit_testing, suffix=".ini") as ini_file:
261
- return Portal(Portal._create_vapp(ini_file))
333
+ return Portal(Portal._create_testapp(ini_file))
262
334
 
263
335
  @staticmethod
264
336
  def create_for_testing_local(ini_file: Optional[str] = None) -> Portal:
265
337
  if isinstance(ini_file, str) and ini_file:
266
- return Portal(Portal._create_vapp(ini_file))
338
+ return Portal(Portal._create_testapp(ini_file))
267
339
  minimal_ini_for_testing_local = "\n".join([
268
340
  "[app:app]\nuse = egg:encoded\nfile_upload_bucket = dummy",
269
341
  "sqlalchemy.url = postgresql://postgres@localhost:5441/postgres?host=/tmp/snovault/pgdata",
@@ -284,11 +356,20 @@ class Portal:
284
356
  "multiauth.policy.auth0.base = encoded.authentication.Auth0AuthenticationPolicy"
285
357
  ])
286
358
  with temporary_file(content=minimal_ini_for_testing_local, suffix=".ini") as minimal_ini_file:
287
- return Portal(Portal._create_vapp(minimal_ini_file))
359
+ return Portal(Portal._create_testapp(minimal_ini_file))
288
360
 
289
361
  @staticmethod
290
- def _create_vapp(value: Union[str, Router, TestApp] = "development.ini", app_name: str = "app") -> TestApp:
291
- if isinstance(value, TestApp):
292
- return value
293
- app = value if isinstance(value, Router) else get_app(value, app_name)
294
- return TestApp(app, {"HTTP_ACCEPT": "application/json", "REMOTE_USER": "TEST"})
362
+ def _create_testapp(arg: Union[TestApp, VirtualApp, Router, str] = None, app_name: Optional[str] = None) -> TestApp:
363
+ if isinstance(arg, TestApp):
364
+ return arg
365
+ elif isinstance(arg, VirtualApp):
366
+ if not isinstance(arg.wrapped_app, TestApp):
367
+ raise Exception("Portal._create_testapp VirtualApp argument error.")
368
+ return arg.wrapped_app
369
+ if isinstance(arg, Router):
370
+ router = arg
371
+ elif isinstance(arg, str) or arg is None:
372
+ router = get_app(arg or "development.ini", app_name or "app")
373
+ else:
374
+ raise Exception("Portal._create_testapp argument error.")
375
+ return TestApp(router, {"HTTP_ACCEPT": "application/json", "REMOTE_USER": "TEST"})
@@ -42,9 +42,9 @@ StructuredDataSet = Type["StructuredDataSet"]
42
42
  class StructuredDataSet:
43
43
 
44
44
  def __init__(self, file: Optional[str] = None, portal: Optional[Union[VirtualApp, TestApp, Portal]] = None,
45
- schemas: Optional[List[dict]] = None, data: Optional[List[dict]] = None,
45
+ schemas: Optional[List[dict]] = None, autoadd: Optional[dict] = None,
46
46
  order: Optional[List[str]] = None, prune: bool = True) -> None:
47
- self.data = {} if not data else data # If portal is None then no schemas nor refs.
47
+ self.data = {}
48
48
  self._portal = Portal(portal, data=self.data, schemas=schemas) if portal else None
49
49
  self._order = order
50
50
  self._prune = prune
@@ -52,13 +52,14 @@ class StructuredDataSet:
52
52
  self._errors = {}
53
53
  self._resolved_refs = set()
54
54
  self._validated = False
55
+ self._autoadd_properties = autoadd if isinstance(autoadd, dict) and autoadd else None
55
56
  self._load_file(file) if file else None
56
57
 
57
58
  @staticmethod
58
59
  def load(file: str, portal: Optional[Union[VirtualApp, TestApp, Portal]] = None,
59
- schemas: Optional[List[dict]] = None,
60
+ schemas: Optional[List[dict]] = None, autoadd: Optional[dict] = None,
60
61
  order: Optional[List[str]] = None, prune: bool = True) -> StructuredDataSet:
61
- return StructuredDataSet(file=file, portal=portal, schemas=schemas, order=order, prune=prune)
62
+ return StructuredDataSet(file=file, portal=portal, schemas=schemas, autoadd=autoadd, order=order, prune=prune)
62
63
 
63
64
  def validate(self, force: bool = False) -> None:
64
65
  if self._validated and not force:
@@ -112,7 +113,7 @@ class StructuredDataSet:
112
113
  def _load_file(self, file: str) -> None:
113
114
  # Returns a dictionary where each property is the name (i.e. the type) of the data,
114
115
  # and the value is array of dictionaries for the data itself. Handle these kinds of files:
115
- # 1. Single CSV of JSON file, where the (base) name of the file is the data type name.
116
+ # 1. Single CSV, TSV, or JSON file, where the (base) name of the file is the data type name.
116
117
  # 2. Single Excel file containing one or more sheets, where each sheet
117
118
  # represents (i.e. is named for, and contains data for) a different type.
118
119
  # 3. Zip file (.zip or .tar.gz or .tgz or .tar), containing data files to load, where the
@@ -163,6 +164,8 @@ class StructuredDataSet:
163
164
  structured_row = structured_row_template.create_row()
164
165
  for column_name, value in row.items():
165
166
  structured_row_template.set_value(structured_row, column_name, value, reader.file, reader.row_number)
167
+ if self._autoadd_properties:
168
+ self._add_properties(structured_row, self._autoadd_properties, schema)
166
169
  self._add(type_name, structured_row)
167
170
  self._note_warning(reader.warnings, "reader")
168
171
  if schema:
@@ -177,6 +180,11 @@ class StructuredDataSet:
177
180
  else:
178
181
  self.data[type_name] = [data] if isinstance(data, dict) else data
179
182
 
183
+ def _add_properties(self, structured_row: dict, properties: dict, schema: Optional[dict] = None) -> None:
184
+ for name in properties:
185
+ if name not in structured_row and (not schema or schema.data.get("properties", {}).get(name)):
186
+ structured_row[name] = properties[name]
187
+
180
188
  def _note_warning(self, item: Optional[Union[dict, List[dict]]], group: str) -> None:
181
189
  self._note_issue(self._warnings, item, group)
182
190
 
@@ -475,7 +483,6 @@ class Schema:
475
483
  if unique:
476
484
  typeinfo[key]["unique"] = True
477
485
  result.update(typeinfo)
478
- # result.update(self._create_typeinfo(array_property_items, parent_key=key))
479
486
  continue
480
487
  result[key] = {"type": property_value_type, "map": self._map_function({**property_value, "column": key})}
481
488
  if ARRAY_NAME_SUFFIX_CHAR in key:
@@ -543,16 +550,13 @@ class Portal(PortalBase):
543
550
 
544
551
  def __init__(self,
545
552
  arg: Optional[Union[VirtualApp, TestApp, Router, Portal, dict, tuple, str]] = None,
546
- env: Optional[str] = None, app: OrchestratedApp = None, server: Optional[str] = None,
547
- key: Optional[Union[dict, tuple]] = None,
548
- portal: Optional[Union[VirtualApp, TestApp, Router, Portal, str]] = None,
549
- data: Optional[dict] = None, schemas: Optional[List[dict]] = None) -> Optional[Portal]:
550
- super().__init__(arg, env=env, app=app, server=server, key=key, portal=portal)
551
- if isinstance(arg, Portal) and not portal:
552
- portal = arg
553
- if isinstance(portal, Portal):
554
- self._schemas = schemas if schemas is not None else portal._schemas # Explicitly specified/known schemas.
555
- self._data = data if data is not None else portal._data # Data set being loaded; e.g. by StructuredDataSet.
553
+ env: Optional[str] = None, server: Optional[str] = None,
554
+ app: Optional[OrchestratedApp] = None,
555
+ data: Optional[dict] = None, schemas: Optional[List[dict]] = None) -> None:
556
+ super().__init__(arg, env=env, server=server, app=app)
557
+ if isinstance(arg, Portal):
558
+ self._schemas = schemas if schemas is not None else arg._schemas # Explicitly specified/known schemas.
559
+ self._data = data if data is not None else arg._data # Data set being loaded; e.g. by StructuredDataSet.
556
560
  else:
557
561
  self._schemas = schemas
558
562
  self._data = data
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dcicutils
3
- Version: 8.5.0.1b3
3
+ Version: 8.5.0.1b6
4
4
  Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
5
5
  Home-page: https://github.com/4dn-dcic/utils
6
6
  License: MIT
@@ -43,7 +43,7 @@ dcicutils/log_utils.py,sha256=7pWMc6vyrorUZQf-V-M3YC6zrPgNhuV_fzm9xqTPph0,10883
43
43
  dcicutils/misc_utils.py,sha256=nRjLEORY35YmJwTjO0fnauBPznaI_bkVasIW8PccDYM,100179
44
44
  dcicutils/obfuscation_utils.py,sha256=fo2jOmDRC6xWpYX49u80bVNisqRRoPskFNX3ymFAmjw,5963
45
45
  dcicutils/opensearch_utils.py,sha256=V2exmFYW8Xl2_pGFixF4I2Cc549Opwe4PhFi5twC0M8,1017
46
- dcicutils/portal_utils.py,sha256=Jp33fgq3wKKYVMQw851EUKgayrQL9_xsm8NKyAP0Vy4,14553
46
+ dcicutils/portal_utils.py,sha256=PLIKONo_BhVrf-r5fkXAaxP5IQZRjvPjf83YxvRx0ZE,18393
47
47
  dcicutils/project_utils.py,sha256=qPdCaFmWUVBJw4rw342iUytwdQC0P-XKpK4mhyIulMM,31250
48
48
  dcicutils/qa_checkers.py,sha256=cdXjeL0jCDFDLT8VR8Px78aS10hwNISOO5G_Zv2TZ6M,20534
49
49
  dcicutils/qa_utils.py,sha256=TT0SiJWiuxYvbsIyhK9VO4uV_suxhB6CpuC4qPacCzQ,160208
@@ -56,14 +56,14 @@ dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19
56
56
  dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
57
57
  dcicutils/snapshot_utils.py,sha256=ymP7PXH6-yEiXAt75w0ldQFciGNqWBClNxC5gfX2FnY,22961
58
58
  dcicutils/ssl_certificate_utils.py,sha256=F0ifz_wnRRN9dfrfsz7aCp4UDLgHEY8LaK7PjnNvrAQ,9707
59
- dcicutils/structured_data.py,sha256=8jEPKeoVOcqhG_i-H6cK4zBXTUFpFi_9ykw5g1h0n4g,32690
59
+ dcicutils/structured_data.py,sha256=mWjH6h9ARUvYMHI4ZiwbaoKtoeM0CAOz1UxCHh4HIHE,32874
60
60
  dcicutils/task_utils.py,sha256=MF8ujmTD6-O2AC2gRGPHyGdUrVKgtr8epT5XU8WtNjk,8082
61
61
  dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
62
62
  dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
63
63
  dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
64
64
  dcicutils/zip_utils.py,sha256=0OXR0aLNwyLIZOzIFTM_5DOun7dxIv6TIZbFiithkO0,3276
65
- dcicutils-8.5.0.1b3.dist-info/LICENSE.txt,sha256=t0_-jIjqxNnymZoNJe-OltRIuuF8qfhN0ATlHyrUJPk,1102
66
- dcicutils-8.5.0.1b3.dist-info/METADATA,sha256=xLOqwEx3paC3otYf4koYSgOZ8WekQIkBVaKZrF-tH84,3314
67
- dcicutils-8.5.0.1b3.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
68
- dcicutils-8.5.0.1b3.dist-info/entry_points.txt,sha256=8wbw5csMIgBXhkwfgsgJeuFcoUc0WsucUxmOyml2aoA,209
69
- dcicutils-8.5.0.1b3.dist-info/RECORD,,
65
+ dcicutils-8.5.0.1b6.dist-info/LICENSE.txt,sha256=t0_-jIjqxNnymZoNJe-OltRIuuF8qfhN0ATlHyrUJPk,1102
66
+ dcicutils-8.5.0.1b6.dist-info/METADATA,sha256=wWiZixknI6Pne33sGS-Lxeh6hSKSScj44tPcctlpHJw,3314
67
+ dcicutils-8.5.0.1b6.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
68
+ dcicutils-8.5.0.1b6.dist-info/entry_points.txt,sha256=8wbw5csMIgBXhkwfgsgJeuFcoUc0WsucUxmOyml2aoA,209
69
+ dcicutils-8.5.0.1b6.dist-info/RECORD,,