henge 0.2.0__py3-none-any.whl → 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- henge/const.py +1 -1
- henge/henge.py +162 -108
- {henge-0.2.0.dist-info → henge-0.2.1.dist-info}/METADATA +1 -1
- henge-0.2.1.dist-info/RECORD +11 -0
- henge-0.2.0.dist-info/RECORD +0 -11
- {henge-0.2.0.dist-info → henge-0.2.1.dist-info}/LICENSE.txt +0 -0
- {henge-0.2.0.dist-info → henge-0.2.1.dist-info}/WHEEL +0 -0
- {henge-0.2.0.dist-info → henge-0.2.1.dist-info}/entry_points.txt +0 -0
- {henge-0.2.0.dist-info → henge-0.2.1.dist-info}/top_level.txt +0 -0
henge/const.py
CHANGED
henge/henge.py
CHANGED
|
@@ -4,7 +4,6 @@ import copy
|
|
|
4
4
|
import hashlib
|
|
5
5
|
import jsonschema
|
|
6
6
|
import logging
|
|
7
|
-
import logmuse
|
|
8
7
|
import json
|
|
9
8
|
import os
|
|
10
9
|
import sys
|
|
@@ -18,10 +17,13 @@ from .const import *
|
|
|
18
17
|
|
|
19
18
|
_LOGGER = logging.getLogger(__name__)
|
|
20
19
|
|
|
20
|
+
|
|
21
21
|
class NotFoundException(Exception):
|
|
22
22
|
"""Raised when a digest is not found"""
|
|
23
|
+
|
|
23
24
|
def __init__(self, m):
|
|
24
25
|
self.message = "{} not found in database".format(m)
|
|
26
|
+
|
|
25
27
|
def __str__(self):
|
|
26
28
|
return self.message
|
|
27
29
|
|
|
@@ -29,27 +31,32 @@ class NotFoundException(Exception):
|
|
|
29
31
|
def md5(seq):
|
|
30
32
|
return hashlib.md5(seq.encode()).hexdigest()
|
|
31
33
|
|
|
34
|
+
|
|
32
35
|
def is_url(maybe_url):
|
|
33
36
|
from urllib.parse import urlparse
|
|
34
|
-
|
|
37
|
+
|
|
38
|
+
return " " not in maybe_url and urlparse(maybe_url).scheme != ""
|
|
39
|
+
|
|
35
40
|
|
|
36
41
|
def read_url(url):
|
|
37
42
|
_LOGGER.info("Reading URL: {}".format(url))
|
|
38
43
|
from urllib.request import urlopen
|
|
39
44
|
from urllib.error import HTTPError
|
|
45
|
+
|
|
40
46
|
try:
|
|
41
47
|
response = urlopen(url)
|
|
42
48
|
except HTTPError as e:
|
|
43
49
|
raise e
|
|
44
|
-
data = response.read()
|
|
45
|
-
text = data.decode(
|
|
50
|
+
data = response.read() # a `bytes` object
|
|
51
|
+
text = data.decode("utf-8")
|
|
46
52
|
print(text)
|
|
47
53
|
return yaml.safe_load(text)
|
|
48
54
|
|
|
49
55
|
|
|
50
56
|
class Henge(object):
|
|
51
|
-
def __init__(
|
|
52
|
-
checksum_function=md5
|
|
57
|
+
def __init__(
|
|
58
|
+
self, database, schemas, schemas_str=[], henges=None, checksum_function=md5
|
|
59
|
+
):
|
|
53
60
|
"""
|
|
54
61
|
A user interface to insert and retrieve decomposable recursive unique
|
|
55
62
|
identifiers (DRUIDs).
|
|
@@ -84,7 +91,9 @@ class Henge(object):
|
|
|
84
91
|
else:
|
|
85
92
|
populated_schemas = []
|
|
86
93
|
if isinstance(schemas, str):
|
|
87
|
-
_LOGGER.error(
|
|
94
|
+
_LOGGER.error(
|
|
95
|
+
"The schemas should be a list. Please pass a list of schemas"
|
|
96
|
+
)
|
|
88
97
|
schemas = [schemas]
|
|
89
98
|
for schema_value in schemas:
|
|
90
99
|
if isinstance(schema_value, str):
|
|
@@ -92,8 +101,10 @@ class Henge(object):
|
|
|
92
101
|
populated_schemas.append(yacman.load_yaml(schema_value))
|
|
93
102
|
elif is_url(schema_value):
|
|
94
103
|
populated_schemas.append(read_url(schema_value))
|
|
95
|
-
else
|
|
96
|
-
_LOGGER.error(
|
|
104
|
+
else:
|
|
105
|
+
_LOGGER.error(
|
|
106
|
+
f"Schema file not found: {schema_value}. Use schemas_str if you meant to specify a direct schema"
|
|
107
|
+
)
|
|
97
108
|
# populated_schemas.append(yaml.safe_load(schema_value))
|
|
98
109
|
|
|
99
110
|
for schema_value in schemas_str:
|
|
@@ -121,7 +132,6 @@ class Henge(object):
|
|
|
121
132
|
self.schemas[item_type] = henge.schemas[item_type]
|
|
122
133
|
self.henges[item_type] = henge
|
|
123
134
|
|
|
124
|
-
|
|
125
135
|
def retrieve(self, druid, reclimit=None, raw=False):
|
|
126
136
|
"""
|
|
127
137
|
Retrieve an item given a digest
|
|
@@ -132,7 +142,11 @@ class Henge(object):
|
|
|
132
142
|
:param bool raw: Return the value as a raw, henge-delimited string, instead
|
|
133
143
|
of processing into a mapping. Default: False.
|
|
134
144
|
"""
|
|
135
|
-
|
|
145
|
+
try:
|
|
146
|
+
item_type = self.database[druid + ITEM_TYPE]
|
|
147
|
+
except KeyError:
|
|
148
|
+
raise NotFoundException(druid)
|
|
149
|
+
|
|
136
150
|
digested_string = self.lookup(druid, item_type)
|
|
137
151
|
reconstructed_item = json.loads(digested_string)
|
|
138
152
|
|
|
@@ -142,30 +156,39 @@ class Henge(object):
|
|
|
142
156
|
reconstructed_item.update(external_values)
|
|
143
157
|
|
|
144
158
|
schema = self.schemas[item_type]
|
|
145
|
-
|
|
159
|
+
|
|
146
160
|
if schema["type"] == "array":
|
|
147
|
-
if isinstance(reclimit, int) and reclimit == 0:
|
|
161
|
+
if isinstance(reclimit, int) and reclimit == 0:
|
|
148
162
|
return reconstructed_item
|
|
149
|
-
if
|
|
150
|
-
_LOGGER.debug(
|
|
163
|
+
if "henge_class" in schema["items"]:
|
|
164
|
+
_LOGGER.debug(
|
|
165
|
+
"Henge classed array: {}; Schema: {}".format(
|
|
166
|
+
digested_string, schema
|
|
167
|
+
)
|
|
168
|
+
)
|
|
151
169
|
if isinstance(reclimit, int):
|
|
152
|
-
reclimit = reclimit - 1
|
|
170
|
+
reclimit = reclimit - 1
|
|
153
171
|
return [self.retrieve(item, reclimit) for item in reconstructed_item]
|
|
154
172
|
elif schema["type"] == "object":
|
|
155
|
-
if
|
|
173
|
+
if "recursive" in schema:
|
|
156
174
|
if isinstance(reclimit, int) and reclimit == 0:
|
|
157
|
-
_LOGGER.debug(
|
|
175
|
+
_LOGGER.debug(
|
|
176
|
+
"Lookup/obj/Recursive: {}; Schema: {}".format(
|
|
177
|
+
digested_string, schema
|
|
178
|
+
)
|
|
179
|
+
)
|
|
158
180
|
return reconstructed_item
|
|
159
181
|
else:
|
|
160
182
|
if isinstance(reclimit, int):
|
|
161
183
|
reclimit = reclimit - 1
|
|
162
|
-
for recursive_attr in schema[
|
|
163
|
-
if
|
|
164
|
-
|
|
184
|
+
for recursive_attr in schema["recursive"]:
|
|
185
|
+
if (
|
|
186
|
+
recursive_attr in reconstructed_item
|
|
187
|
+
and reconstructed_item[recursive_attr] != ""
|
|
188
|
+
):
|
|
165
189
|
reconstructed_item[recursive_attr] = self.retrieve(
|
|
166
|
-
reconstructed_item[recursive_attr],
|
|
167
|
-
|
|
168
|
-
raw)
|
|
190
|
+
reconstructed_item[recursive_attr], reclimit, raw
|
|
191
|
+
)
|
|
169
192
|
return reconstructed_item
|
|
170
193
|
|
|
171
194
|
def lookup(self, druid, item_type):
|
|
@@ -173,7 +196,7 @@ class Henge(object):
|
|
|
173
196
|
henge_to_query = self.henges[item_type]
|
|
174
197
|
except:
|
|
175
198
|
_LOGGER.debug("No henges available for this item type")
|
|
176
|
-
raise NotFoundException(druid)
|
|
199
|
+
raise NotFoundException(druid)
|
|
177
200
|
try:
|
|
178
201
|
string = henge_to_query.database[druid]
|
|
179
202
|
except KeyError:
|
|
@@ -181,7 +204,6 @@ class Henge(object):
|
|
|
181
204
|
|
|
182
205
|
return string
|
|
183
206
|
|
|
184
|
-
|
|
185
207
|
@property
|
|
186
208
|
def item_types(self):
|
|
187
209
|
"""
|
|
@@ -217,20 +239,19 @@ class Henge(object):
|
|
|
217
239
|
fits.
|
|
218
240
|
"""
|
|
219
241
|
|
|
220
|
-
|
|
221
242
|
_LOGGER.debug("Insert type: {} / Item: {}".format(item_type, item))
|
|
222
|
-
|
|
243
|
+
|
|
223
244
|
if item_type not in self.schemas.keys():
|
|
224
|
-
_LOGGER.error(
|
|
225
|
-
|
|
226
|
-
|
|
245
|
+
_LOGGER.error(
|
|
246
|
+
"I don't know about items of type '{}'. "
|
|
247
|
+
"I know of: '{}'".format(item_type, list(self.schemas.keys()))
|
|
248
|
+
)
|
|
227
249
|
return False
|
|
228
250
|
|
|
229
251
|
schema = self.schemas[item_type]
|
|
230
252
|
|
|
231
|
-
|
|
232
253
|
flat_item = item
|
|
233
|
-
if schema[
|
|
254
|
+
if schema["type"] == "object":
|
|
234
255
|
flat_item = {}
|
|
235
256
|
if isinstance(reclimit, int) and reclimit == 0:
|
|
236
257
|
return self._insert_flat(item, item_type)
|
|
@@ -239,41 +260,51 @@ class Henge(object):
|
|
|
239
260
|
reclimit = reclimit - 1
|
|
240
261
|
for prop in item:
|
|
241
262
|
if prop in schema["properties"]:
|
|
242
|
-
_LOGGER.debug(
|
|
263
|
+
_LOGGER.debug(
|
|
264
|
+
"-Prop {}; Schema: {}".format(
|
|
265
|
+
prop, str(schema["properties"][prop])
|
|
266
|
+
)
|
|
267
|
+
)
|
|
243
268
|
if "recursive" in schema and prop in schema["recursive"]:
|
|
244
269
|
hclass = schema["properties"][prop]["henge_class"]
|
|
245
270
|
digest = self.insert(item[prop], hclass, reclimit)
|
|
246
271
|
flat_item[prop] = digest
|
|
247
|
-
elif schema["properties"][prop]["type"] in [
|
|
272
|
+
elif schema["properties"][prop]["type"] in ["array"]:
|
|
248
273
|
digest = self.insert(item[prop], "array", reclimit)
|
|
249
274
|
flat_item[prop] = digest
|
|
250
275
|
else:
|
|
251
276
|
flat_item[prop] = item[prop]
|
|
252
|
-
_LOGGER.debug(
|
|
277
|
+
_LOGGER.debug(
|
|
278
|
+
"Prop: {}; Flat item: {}".format(prop, flat_item[prop])
|
|
279
|
+
)
|
|
253
280
|
else:
|
|
254
281
|
_LOGGER.debug(f"Prop: {prop}. Ignoring due to not in schema")
|
|
255
282
|
pass # Ignore non-schema defined properties
|
|
256
283
|
|
|
257
284
|
# if len(flat_item) == 0:
|
|
258
285
|
# flat_item = item
|
|
259
|
-
elif schema[
|
|
286
|
+
elif schema["type"] == "array":
|
|
260
287
|
flat_item = []
|
|
261
|
-
if
|
|
288
|
+
if "henge_class" in schema["items"]:
|
|
262
289
|
digest = []
|
|
263
|
-
hclass = schema[
|
|
290
|
+
hclass = schema["items"]["henge_class"]
|
|
264
291
|
if isinstance(reclimit, int) and reclimit == 0:
|
|
265
292
|
return self._insert_flat(item, item_type)
|
|
266
293
|
else:
|
|
267
294
|
if isinstance(reclimit, int):
|
|
268
|
-
reclimit = reclimit - 1
|
|
269
|
-
_LOGGER.debug(
|
|
295
|
+
reclimit = reclimit - 1
|
|
296
|
+
_LOGGER.debug(
|
|
297
|
+
"Item: {}. Pyclass: {}. hclass: {}".format(
|
|
298
|
+
item, type(item), hclass
|
|
299
|
+
)
|
|
300
|
+
)
|
|
270
301
|
for element in item:
|
|
271
302
|
digest.append(self.insert(element, hclass, reclimit))
|
|
272
303
|
flat_item = digest
|
|
273
304
|
else:
|
|
274
305
|
flat_item = item
|
|
275
306
|
_LOGGER.debug("Array flat item: {}".format(flat_item))
|
|
276
|
-
else:
|
|
307
|
+
else: # A primitive type with a henge class
|
|
277
308
|
_LOGGER.debug("Nice! You're using a henge-classed primitive type!")
|
|
278
309
|
hclass = schema["henge_class"]
|
|
279
310
|
# digest = self.insert(item, hclass)
|
|
@@ -281,12 +312,11 @@ class Henge(object):
|
|
|
281
312
|
|
|
282
313
|
return self._insert_flat(flat_item, item_type)
|
|
283
314
|
|
|
284
|
-
|
|
285
315
|
def _insert_flat(self, item, item_type=None, item_name=None):
|
|
286
316
|
"""
|
|
287
317
|
Add flattened items (of a specified type) to the database.
|
|
288
318
|
|
|
289
|
-
Flattened items have removed all levels, so it's only attributes and
|
|
319
|
+
Flattened items have removed all levels, so it's only attributes and
|
|
290
320
|
strict values; no nesting allowed. Use the upstream insert function
|
|
291
321
|
to insert full structured objects, which calls this function.
|
|
292
322
|
|
|
@@ -297,9 +327,10 @@ class Henge(object):
|
|
|
297
327
|
fits.
|
|
298
328
|
"""
|
|
299
329
|
if item_type not in self.schemas.keys():
|
|
300
|
-
_LOGGER.error(
|
|
301
|
-
|
|
302
|
-
|
|
330
|
+
_LOGGER.error(
|
|
331
|
+
"I don't know about items of type '{}'. "
|
|
332
|
+
"I know of: '{}'".format(item_type, list(self.schemas.keys()))
|
|
333
|
+
)
|
|
303
334
|
return False
|
|
304
335
|
|
|
305
336
|
# digest_version should be automatically appended to the item by the
|
|
@@ -310,11 +341,14 @@ class Henge(object):
|
|
|
310
341
|
|
|
311
342
|
valid_schema = self.schemas[item_type]
|
|
312
343
|
# Add defaults here ?
|
|
313
|
-
try:
|
|
344
|
+
try:
|
|
314
345
|
jsonschema.validate(item, valid_schema)
|
|
315
346
|
except jsonschema.ValidationError as e:
|
|
316
|
-
_LOGGER.error(
|
|
317
|
-
format(
|
|
347
|
+
_LOGGER.error(
|
|
348
|
+
"Not valid data. Item type: {}. Attempting to insert item: {}".format(
|
|
349
|
+
item_type, item
|
|
350
|
+
)
|
|
351
|
+
)
|
|
318
352
|
print(e)
|
|
319
353
|
|
|
320
354
|
if isinstance(item, str):
|
|
@@ -322,7 +356,9 @@ class Henge(object):
|
|
|
322
356
|
try:
|
|
323
357
|
existing_item_type = henge_to_query.database[item + ITEM_TYPE]
|
|
324
358
|
except KeyError:
|
|
325
|
-
_LOGGER.error(
|
|
359
|
+
_LOGGER.error(
|
|
360
|
+
"If you're trying to insert an item with druids, the sub-items must exist in the database."
|
|
361
|
+
)
|
|
326
362
|
# return None
|
|
327
363
|
try:
|
|
328
364
|
existing_item = henge_to_query.database[item]
|
|
@@ -330,13 +366,13 @@ class Henge(object):
|
|
|
330
366
|
_LOGGER.error("That item wasn't in the database.")
|
|
331
367
|
|
|
332
368
|
# if (item_type == existing_item_type):
|
|
333
|
-
|
|
369
|
+
# _LOGGER.info("But wait!!! That's already here, and it's great! I'll return that!")
|
|
334
370
|
return item
|
|
335
371
|
|
|
336
372
|
raise e
|
|
337
373
|
return None
|
|
338
|
-
|
|
339
|
-
_LOGGER.debug(f"item to insert: {item}")
|
|
374
|
+
|
|
375
|
+
_LOGGER.debug(f"item to insert: {item}")
|
|
340
376
|
item_inherent_split = select_inherent_properties(item, valid_schema)
|
|
341
377
|
attr_string = canonical_str(item_inherent_split["inherent"])
|
|
342
378
|
external_string = canonical_str(item_inherent_split["external"])
|
|
@@ -346,11 +382,16 @@ class Henge(object):
|
|
|
346
382
|
druid = self.checksum_function(attr_string)
|
|
347
383
|
self._henge_insert(druid, attr_string, item_type, external_string)
|
|
348
384
|
|
|
349
|
-
_LOGGER.debug(
|
|
350
|
-
|
|
385
|
+
_LOGGER.debug(
|
|
386
|
+
"Inserted flat item. Digest: {} / Type: {} / Item: {}".format(
|
|
387
|
+
druid, item_type, item
|
|
388
|
+
)
|
|
389
|
+
)
|
|
351
390
|
return druid
|
|
352
391
|
|
|
353
|
-
def _henge_insert(
|
|
392
|
+
def _henge_insert(
|
|
393
|
+
self, druid, string, item_type, external_string, digest_version=None
|
|
394
|
+
):
|
|
354
395
|
"""
|
|
355
396
|
Inserts an item into the database, with henge-metadata slots for item
|
|
356
397
|
type and digest version.
|
|
@@ -392,7 +433,7 @@ class Henge(object):
|
|
|
392
433
|
except (KeyError, AttributeError):
|
|
393
434
|
pass
|
|
394
435
|
except AttributeError as e:
|
|
395
|
-
_LOGGER.warn(f"Error trying to iterate over database items: {e}")
|
|
436
|
+
_LOGGER.warn(f"Error trying to iterate over database items: {e}")
|
|
396
437
|
|
|
397
438
|
def show(self):
|
|
398
439
|
"""
|
|
@@ -401,7 +442,6 @@ class Henge(object):
|
|
|
401
442
|
for k, v in self.database.items():
|
|
402
443
|
print(k, v)
|
|
403
444
|
|
|
404
|
-
|
|
405
445
|
def __repr__(self):
|
|
406
446
|
repr = "Henge object. Item types: " + ",".join(self.item_types)
|
|
407
447
|
return repr
|
|
@@ -413,68 +453,74 @@ def split_schema(schema, name=None):
|
|
|
413
453
|
"""
|
|
414
454
|
slist = {}
|
|
415
455
|
# base case
|
|
416
|
-
if schema[
|
|
456
|
+
if schema["type"] not in ["object", "array"]:
|
|
417
457
|
_LOGGER.debug(schema)
|
|
418
458
|
if name:
|
|
419
459
|
slist[name] = schema
|
|
420
|
-
elif
|
|
421
|
-
slist[schema[
|
|
460
|
+
elif "henge_class" in schema:
|
|
461
|
+
slist[schema["henge_class"]] = schema
|
|
422
462
|
_LOGGER.debug("Returning slist: {}".format(str(slist)))
|
|
423
463
|
return slist
|
|
424
|
-
elif schema[
|
|
464
|
+
elif schema["type"] == "object":
|
|
425
465
|
recursive_properties = []
|
|
426
|
-
if
|
|
466
|
+
if "henge_class" in schema:
|
|
427
467
|
schema_copy = copy.deepcopy(schema)
|
|
428
|
-
_LOGGER.debug("adding " + str(schema_copy[
|
|
429
|
-
henge_class = schema_copy[
|
|
468
|
+
_LOGGER.debug("adding " + str(schema_copy["henge_class"]))
|
|
469
|
+
henge_class = schema_copy["henge_class"]
|
|
430
470
|
# del schema_copy['henge_class']
|
|
431
|
-
for p in schema_copy[
|
|
471
|
+
for p in schema_copy["properties"]:
|
|
432
472
|
hclass = None
|
|
433
|
-
if
|
|
434
|
-
hclass = schema_copy[
|
|
473
|
+
if "henge_class" in schema_copy["properties"][p]:
|
|
474
|
+
hclass = schema_copy["properties"][p]["henge_class"]
|
|
435
475
|
recursive_properties.append(p)
|
|
436
|
-
if schema_copy[
|
|
476
|
+
if schema_copy["properties"][p]["type"] in ["object"]:
|
|
437
477
|
# recursive_properties.append(p)
|
|
438
|
-
schema_copy[
|
|
478
|
+
schema_copy["properties"][p] = {"type": "string"}
|
|
439
479
|
if hclass:
|
|
440
|
-
schema_copy[
|
|
441
|
-
if schema_copy[
|
|
480
|
+
schema_copy["properties"][p]["henge_class"] = hclass
|
|
481
|
+
if schema_copy["properties"][p]["type"] in ["array"]:
|
|
442
482
|
# recursive_properties.append(p)
|
|
443
|
-
schema_copy[
|
|
483
|
+
if schema_copy["properties"][p]["items"]["type"] == "integer":
|
|
484
|
+
schema_copy["properties"][p] = {"type": "string"}
|
|
485
|
+
else:
|
|
486
|
+
schema_copy["properties"][p] = {"type": "string"}
|
|
444
487
|
if hclass:
|
|
445
|
-
schema_copy[
|
|
488
|
+
schema_copy["properties"][p]["henge_class"] = hclass
|
|
446
489
|
else:
|
|
447
|
-
schema_copy[
|
|
490
|
+
schema_copy["properties"][p]["henge_class"] = "strarray"
|
|
448
491
|
# schema_copy['properties'][p]['type'] = "string"
|
|
449
492
|
# del schema_copy['properties']
|
|
450
|
-
_LOGGER.debug(
|
|
451
|
-
|
|
493
|
+
_LOGGER.debug(
|
|
494
|
+
"Adding recursive properties: {}".format(recursive_properties)
|
|
495
|
+
)
|
|
496
|
+
schema_copy["recursive"] = recursive_properties
|
|
452
497
|
slist[henge_class] = schema_copy
|
|
453
498
|
|
|
454
|
-
for p in schema[
|
|
499
|
+
for p in schema["properties"]:
|
|
455
500
|
# if schema['properties'][p]['type'] in ['object', 'array']:
|
|
456
|
-
# recursive_properties.append(p)
|
|
457
|
-
schema_sub = schema[
|
|
501
|
+
# recursive_properties.append(p)
|
|
502
|
+
schema_sub = schema["properties"][p]
|
|
458
503
|
_LOGGER.debug("checking property:" + p)
|
|
459
|
-
slist.update(split_schema(schema[
|
|
460
|
-
elif schema[
|
|
504
|
+
slist.update(split_schema(schema["properties"][p]))
|
|
505
|
+
elif schema["type"] == "array":
|
|
461
506
|
_LOGGER.debug("found array")
|
|
462
507
|
_LOGGER.debug(schema)
|
|
463
|
-
if
|
|
508
|
+
if "henge_class" in schema:
|
|
464
509
|
schema_copy = copy.deepcopy(schema)
|
|
465
|
-
_LOGGER.debug("adding " + str(schema[
|
|
466
|
-
henge_class = schema_copy[
|
|
510
|
+
_LOGGER.debug("adding " + str(schema["henge_class"]))
|
|
511
|
+
henge_class = schema_copy["henge_class"]
|
|
467
512
|
# del schema_copy['henge_class']
|
|
468
|
-
schema_copy[
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
513
|
+
if schema_copy["items"]["type"] != "integer":
|
|
514
|
+
schema_copy["items"] = {"type": "string"}
|
|
515
|
+
if "recursive" in schema_copy and schema_copy["recursive"]:
|
|
516
|
+
schema_copy["items"]["recursive"] = True
|
|
517
|
+
if "henge_class" in schema["items"]:
|
|
518
|
+
schema_copy["items"]["henge_class"] = schema["items"]["henge_class"]
|
|
473
519
|
# schema_copy['items']['type'] = "string"
|
|
474
520
|
# if 'properties' in schema_copy['items']:
|
|
475
521
|
# del schema_copy['items']['properties']
|
|
476
522
|
slist[henge_class] = schema_copy
|
|
477
|
-
schema_sub = schema[
|
|
523
|
+
schema_sub = schema["items"]
|
|
478
524
|
slist.update(split_schema(schema_sub))
|
|
479
525
|
else:
|
|
480
526
|
_LOGGER.debug("Classless array")
|
|
@@ -486,8 +532,11 @@ def split_schema(schema, name=None):
|
|
|
486
532
|
|
|
487
533
|
|
|
488
534
|
def canonical_str(item: dict) -> str:
|
|
489
|
-
"""
|
|
490
|
-
return json.dumps(
|
|
535
|
+
"""Convert a dict into a canonical string representation"""
|
|
536
|
+
return json.dumps(
|
|
537
|
+
item, separators=(",", ":"), ensure_ascii=False, allow_nan=False, sort_keys=True
|
|
538
|
+
)
|
|
539
|
+
|
|
491
540
|
|
|
492
541
|
def select_inherent_properties(item: dict, schema: dict) -> dict:
|
|
493
542
|
if schema["type"] == "object":
|
|
@@ -503,25 +552,25 @@ def select_inherent_properties(item: dict, schema: dict) -> dict:
|
|
|
503
552
|
return {"inherent": item, "external": None}
|
|
504
553
|
|
|
505
554
|
|
|
506
|
-
|
|
507
555
|
def is_schema_recursive(schema):
|
|
508
556
|
"""
|
|
509
557
|
Determine if a given schema has elements that need to recurse
|
|
510
558
|
"""
|
|
511
559
|
# return 'recursive' in schema # old way
|
|
512
560
|
is_recursive = False
|
|
513
|
-
if schema[
|
|
514
|
-
for prop in schema[
|
|
515
|
-
if schema[
|
|
561
|
+
if schema["type"] == "object":
|
|
562
|
+
for prop in schema["properties"]:
|
|
563
|
+
if schema["properties"]["prop"]["type"] in ["object", "array"]:
|
|
516
564
|
return True
|
|
517
|
-
if schema[
|
|
518
|
-
if schema[
|
|
565
|
+
if schema["type"] == "array":
|
|
566
|
+
if schema["items"]["type"] in ["object", "array"]:
|
|
519
567
|
return True
|
|
520
568
|
return False
|
|
521
569
|
|
|
522
570
|
|
|
523
|
-
def connect_mongo(
|
|
524
|
-
|
|
571
|
+
def connect_mongo(
|
|
572
|
+
host="0.0.0.0", port=27017, database="henge_dict", collection="store"
|
|
573
|
+
):
|
|
525
574
|
"""
|
|
526
575
|
Connect to MongoDB and return the MongoDB-backed dict object
|
|
527
576
|
|
|
@@ -536,15 +585,20 @@ def connect_mongo(host='0.0.0.0', port=27017, database='henge_dict',
|
|
|
536
585
|
"""
|
|
537
586
|
from importlib import import_module
|
|
538
587
|
from inspect import stack
|
|
588
|
+
|
|
539
589
|
for lib in LIBS_BY_BACKEND["mongo"]:
|
|
540
590
|
try:
|
|
541
591
|
globals()[lib] = import_module(lib)
|
|
542
592
|
except ImportError:
|
|
543
593
|
raise ImportError(
|
|
544
594
|
"Requirements not met. Package '{}' is required to setup "
|
|
545
|
-
"MongoDB connection. Install the package and call '{}' again.".
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
595
|
+
"MongoDB connection. Install the package and call '{}' again.".format(
|
|
596
|
+
lib, stack()[0][3]
|
|
597
|
+
)
|
|
598
|
+
)
|
|
599
|
+
pymongo.Connection = lambda host, port, **kwargs: pymongo.MongoClient(
|
|
600
|
+
host=host, port=port
|
|
601
|
+
)
|
|
602
|
+
return mongodict.MongoDict(
|
|
603
|
+
host=host, port=port, database=database, collection=collection
|
|
604
|
+
)
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
henge/__init__.py,sha256=Su4ZQre-GS24y-LjiAIf57GYovLTww0hkk2fDxzMt_g,289
|
|
2
|
+
henge/_version.py,sha256=Zn1KFblwuFHiDRdRAiRnDBRkbPttWh44jKa5zG2ov0E,22
|
|
3
|
+
henge/const.py,sha256=0t3EgqdjmKBd-zu5L8AJnGoWv0T3sAtvXf-3b62Dd-Y,194
|
|
4
|
+
henge/deprecated.py,sha256=C8eINR2gWCiNaN2b3gbFYn8jfJ0ftJm8a_fIgVVVzXc,11248
|
|
5
|
+
henge/henge.py,sha256=mvAcXefMy5r8XcgRe9XXPH5uuXdyL9-sXRYWLNp_fos,23454
|
|
6
|
+
henge-0.2.1.dist-info/LICENSE.txt,sha256=oB6ZGDa4kcznznJKJsLLFFcOZyi8Y6e2Jv0rJozgp-I,1269
|
|
7
|
+
henge-0.2.1.dist-info/METADATA,sha256=zW6QW4rlOKOpkymjUGkvlMRhbdXZQU_7YvTx3WM2ap8,1270
|
|
8
|
+
henge-0.2.1.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
|
|
9
|
+
henge-0.2.1.dist-info/entry_points.txt,sha256=c2OKgrH1a5Cx2osbUFSe9NFK8CbN82lPPsi4wry77_M,61
|
|
10
|
+
henge-0.2.1.dist-info/top_level.txt,sha256=QyovlLuKhhKP1r8bMVmxLdke9F6PZFIN7VlkzvB0xIQ,6
|
|
11
|
+
henge-0.2.1.dist-info/RECORD,,
|
henge-0.2.0.dist-info/RECORD
DELETED
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
henge/__init__.py,sha256=Su4ZQre-GS24y-LjiAIf57GYovLTww0hkk2fDxzMt_g,289
|
|
2
|
-
henge/_version.py,sha256=Zn1KFblwuFHiDRdRAiRnDBRkbPttWh44jKa5zG2ov0E,22
|
|
3
|
-
henge/const.py,sha256=BW_CMh8eU7QR8umjZA3Th3uNHZNi7jmbDp9J_uHg5to,193
|
|
4
|
-
henge/deprecated.py,sha256=C8eINR2gWCiNaN2b3gbFYn8jfJ0ftJm8a_fIgVVVzXc,11248
|
|
5
|
-
henge/henge.py,sha256=Cq_a7EVF519yq2Sj_QGE6GvWyJkaIITvJZ6hvblX-GU,22577
|
|
6
|
-
henge-0.2.0.dist-info/LICENSE.txt,sha256=oB6ZGDa4kcznznJKJsLLFFcOZyi8Y6e2Jv0rJozgp-I,1269
|
|
7
|
-
henge-0.2.0.dist-info/METADATA,sha256=asBCHFYvf83mH50QWSpaxmX6LEQCv7RKY0PtYq7P_6c,1270
|
|
8
|
-
henge-0.2.0.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
|
|
9
|
-
henge-0.2.0.dist-info/entry_points.txt,sha256=c2OKgrH1a5Cx2osbUFSe9NFK8CbN82lPPsi4wry77_M,61
|
|
10
|
-
henge-0.2.0.dist-info/top_level.txt,sha256=QyovlLuKhhKP1r8bMVmxLdke9F6PZFIN7VlkzvB0xIQ,6
|
|
11
|
-
henge-0.2.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|