henge 0.1.1__py3-none-any.whl → 0.2.1__py3-none-any.whl

henge/__init__.py

@@ -5,6 +5,6 @@ from ._version import __version__
 from .henge import *
 
 __classes__ = ["Henge"]
-__all__ = __classes__ + ["connect_mongo", "split_schema", "NotFoundException"]
+__all__ = __classes__ + ["connect_mongo", "split_schema", "NotFoundException", "canonical_str"]
 
 logmuse.init_logger("henge", propagate=True)

henge/_version.py

@@ -1 +1 @@
-__version__ = "0.1.1"
+__version__ = "0.2.0"

henge/const.py

@@ -1,4 +1,4 @@
 LIBS_BY_BACKEND = {"mongo": ["pymongo", "mongodict"]}
 DELIM_ATTR = ","  # chr(30); separating attributes in an item
 DELIM_ITEM = ","  # separating items in a collection
-ITEM_TYPE = "_item_type"
+ITEM_TYPE = "_item_type"

henge/deprecated.py

@@ -0,0 +1,224 @@
+
+    def retrieveOld(self, druid, reclimit=None, raw=False):
+
+        try:
+            item_type = self.database[druid + ITEM_TYPE]
+        except:
+            _LOGGER.debug(f"Item type not saved in database for {druid}")
+            raise NotFoundException(druid)
+            
+        # _LOGGER.debug("item_type: {}".format(item_type))
+        # _LOGGER.debug("henge_to_query: {}".format(henge_to_query))
+
+        schema = self.schemas[item_type] #"type" in schema and 
+        # string = druid
+        _LOGGER.debug("Got druid to retrieve: {} / item_type: {} / schema: {}".format(
+            druid, item_type, schema))
+
+        if schema["type"] == "array":
+            string = self.lookup(druid, item_type)
+            _LOGGER.debug("Lookup/array/Recursive: {}; Schema: {}".format(string, schema))
+            splitstr = string.split(DELIM_ITEM)
+            # if self.flexible_digests:
+            #     pass
+            #     item_name = splitstr.pop(0)
+            if isinstance(reclimit, int) and reclimit == 0:  
+                return splitstr
+            if 'henge_class' in schema['items']:
+                _LOGGER.debug("Henge classed array: {}; Schema: {}".format(string, schema))
+                if isinstance(reclimit, int):
+                    reclimit = reclimit - 1                
+                return [self.retrieve(substr, reclimit) for substr in splitstr]
+            else:
+                return splitstr
+        elif schema["type"] == "object":
+            string = self.lookup(druid, item_type)
+            attr_array = string.split(DELIM_ATTR)
+            if self.flexible_digests:
+                keys = attr_array[::2]  # evens
+                vals = attr_array[1::2]  # odds
+                item_reconstituted = dict(zip(keys,vals))    
+            else:
+                item_reconstituted = dict(zip(schema['properties'].keys(),
+                                          attr_array))
+            # I think this part needs to be removed... it's based on the
+            # previous 'recursive' for arrays, which went away...
+            # but actually these may be added in by me, so nevermind.
+            if 'recursive' in schema:
+                if isinstance(reclimit, int) and reclimit == 0:
+                    _LOGGER.debug("Lookup/obj/Recursive: {}; Schema: {}".format(string, schema))
+                    return item_reconstituted
+                else:
+                    if isinstance(reclimit, int):
+                        reclimit = reclimit - 1
+                    for recursive_attr in schema['recursive']:                    
+                        if recursive_attr in item_reconstituted \
+                                and item_reconstituted[recursive_attr] != "":
+                            item_reconstituted[recursive_attr] = self.retrieve(
+                                item_reconstituted[recursive_attr],
+                                reclimit,
+                                raw)                
+            return item_reconstituted
+        else: # It must be a primitive type
+            # but it could be a primitive (string) that represents something to lookup,
+            # or something not-to-lookup (or already looked up)
+            _LOGGER.debug("Lookup/prim: {}; Schema: {}".format(druid, schema))
+            # return string
+            if 'henge_class' in schema and self.schemas[schema['henge_class']]['type'] in ['object', 'array']:
+                if isinstance(reclimit, int) and reclimit == 0:
+                    _LOGGER.debug("Lookup/prim/Recursive-skip: {}; Schema: {}".format(string, schema))
+                    string = self.lookup(druid, item_type)
+                    return string
+                else:
+                    if isinstance(reclimit, int):
+                        reclimit = reclimit - 1
+                    _LOGGER.debug("Lookup/prim/Recursive: {}; Schema: {}".format(druid, schema))
+                    return self.retrieve(druid, reclimit, raw)
+            else:
+                string = self.lookup(druid, item_type)
+                _LOGGER.debug("Lookup/prim/Non-recursive: {}; Schema: {}".format(string, schema))
+                return string #self.retrieve(string, reclimit, raw)      
+
+        # try:
+        #     string = henge_to_query.database[druid]
+        # except KeyError:
+        #     raise NotFoundException(druid)
+
+        # return reconstruct_item(string, schema, reclimit)
+
+   def retrieve2(self, druid, reclimit=None, raw=False):
+        """
+        Retrieve an item given a digest
+
+        :param str druid: The Decomposable recursive unique identifier (DRUID), or
+            digest that uniquely identifies that item to retrieve.
+        :param int reclimit: Recursion limit. Set to None for no limit (default).
+        :param bool raw: Return the value as a raw, henge-delimited string, instead
+            of processing into a mapping. Default: False.
+        """
+        def reconstruct_item(string, schema, reclimit):
+            if "type" in schema and schema["type"] == "array":
+                _LOGGER.debug("Lookup/array/Recursive: {}; Schema: {}".format(string, schema))
+                splitstr = string.split(DELIM_ITEM)
+                # if self.flexible_digests:
+                #     pass
+                #     item_name = splitstr.pop(0)
+                if 'henge_class' in schema['items'] and schema['items']['type'] not in ["object", "array"]:
+                    _LOGGER.debug("Henge classed array: {}; Schema: {}".format(string, schema))
+                    return "ASDF"
+                    return [reconstruct_item(self.henges[item_type].database[substr], schema["items"], reclimit)
+                        for substr in splitstr]
+                else:
+                    return [reconstruct_item(substr, schema["items"], reclimit)
+                        for substr in splitstr]
+            elif schema["type"] == "object":
+                attr_array = string.split(DELIM_ATTR)
+                if self.flexible_digests:
+                    keys = attr_array[::2]  # evens
+                    vals = attr_array[1::2]  # odds
+                    item_reconstituted = dict(zip(keys,vals))    
+                else:
+                    item_reconstituted = dict(zip(schema['properties'].keys(),
+                                              attr_array))
+                # I think this part needs to be removed... it's based on the
+                # previous 'recursive' for arrays, which went away...
+                # but actually these may be added in by me, so nevermind.
+                if 'recursive' in schema:
+                    if isinstance(reclimit, int) and reclimit == 0:
+                        _LOGGER.debug("Lookup/obj/Recursive: {}; Schema: {}".format(string, schema))
+                        return item_reconstituted
+                    else:
+                        if isinstance(reclimit, int):
+                            reclimit = reclimit - 1
+                        for recursive_attr in schema['recursive']:                    
+                            if item_reconstituted[recursive_attr] \
+                                    and item_reconstituted[recursive_attr] != "":
+                                item_reconstituted[recursive_attr] = self.retrieve(
+                                    item_reconstituted[recursive_attr],
+                                    reclimit,
+                                    raw)                
+                return item_reconstituted
+            else: # it must be a primitive
+                # but it could be a primitive (string) that represents something to lookup,
+                # or something not-to-lookup (or already looked up)
+                _LOGGER.debug("Lookup/prim: {}; Schema: {}".format(string, schema))
+                # return string
+                if 'henge_class' in schema and self.schemas[schema['henge_class']]['type'] in ['object', 'array']:
+                    if isinstance(reclimit, int) and reclimit == 0:
+                        _LOGGER.debug("Lookup/prim/Recursive-skip: {}; Schema: {}".format(string, schema))
+                        return string
+                    else:
+                        if isinstance(reclimit, int):
+                            reclimit = reclimit - 1
+                        _LOGGER.debug("Lookup/prim/Recursive: {}; Schema: {}".format(string, schema))
+                        return self.retrieve(string, reclimit, raw)
+                else:
+                    _LOGGER.debug("Lookup/prim/Non-recursive: {}; Schema: {}".format(string, schema))
+                    return string
+
+        # This requires the database to have __iter__ defined...and it scrolls through
+        # not a great way, take it out! 2021-01 NS
+        # I'll instead do a try block
+        # if not druid + ITEM_TYPE in self.database:
+            # raise NotFoundException(druid)
+
+        try:
+            item_type = self.database[druid + ITEM_TYPE]
+        except:
+            _LOGGER.debug(f"Item type not saved in database for {druid}")
+            raise NotFoundException(druid)
+            
+        try:
+            henge_to_query = self.henges[item_type]
+        except:
+            _LOGGER.debug("No henges available for this item type")
+            raise NotFoundException(druid)
+        # _LOGGER.debug("item_type: {}".format(item_type))
+        # _LOGGER.debug("henge_to_query: {}".format(henge_to_query))
+        try:
+            string = henge_to_query.database[druid]
+        except KeyError:
+            raise NotFoundException(druid)
+
+        schema = self.schemas[item_type]
+        _LOGGER.debug("Got druid to retrieve: {} / item_type: {} / schema: {}".format(
+            druid, item_type, schema))
+        return reconstruct_item(string, schema, reclimit)
+
+
+
+Part of: _insert_flat
+
+        def safestr(item, x):
+            try:
+                return str(item[x])
+            except (ValueError, TypeError, KeyError):
+                return ""
+
+
+
+        def build_attr_string(item, schema, item_name=None):
+            if "type" in schema and schema["type"] == "array":
+                if self.flexible_digests:
+                    return DELIM_ITEM.join([build_attr_string(x, schema['items'])
+                                        for x in item])
+                else:
+                    return DELIM_ITEM.join([build_attr_string(x, schema['items'])
+                                        for x in item])
+            elif schema["type"] == "object" and 'properties' in schema:
+                if self.flexible_digests:
+                    # flexible schema
+                    keys_to_include = sorted([x for x in item.keys() if x in list(schema['properties'].keys())])
+                    return DELIM_ATTR.join([DELIM_ATTR.join([k, safestr(item, k)]) for k in keys_to_include])
+
+                else:
+                    # fixed schema
+                    return DELIM_ATTR.join([safestr(item, x) for x in
+                                        list(schema['properties'].keys())])
+            else: #assume it's a primitive
+                if self.flexible_digests:
+                    return item
+                    attr_string = DELIM_ATTR.join([item_name, item])
+                    return attr_string
+                else:
+                    return item

henge/henge.py

@@ -4,7 +4,7 @@ import copy
 import hashlib
 import jsonschema
 import logging
-import logmuse
+import json
 import os
 import sys
 import yacman
@@ -17,10 +17,13 @@ from .const import *
 
 _LOGGER = logging.getLogger(__name__)
 
+
 class NotFoundException(Exception):
     """Raised when a digest is not found"""
+
     def __init__(self, m):
         self.message = "{} not found in database".format(m)
+
     def __str__(self):
         return self.message
 
@@ -28,27 +31,32 @@ class NotFoundException(Exception):
 def md5(seq):
     return hashlib.md5(seq.encode()).hexdigest()
 
+
 def is_url(maybe_url):
     from urllib.parse import urlparse
-    return ' ' not in maybe_url and urlparse(maybe_url).scheme != ""
+
+    return " " not in maybe_url and urlparse(maybe_url).scheme != ""
+
 
 def read_url(url):
     _LOGGER.info("Reading URL: {}".format(url))
     from urllib.request import urlopen
     from urllib.error import HTTPError
+
     try:
         response = urlopen(url)
     except HTTPError as e:
         raise e
-    data = response.read()      # a `bytes` object
-    text = data.decode('utf-8')
+    data = response.read()  # a `bytes` object
+    text = data.decode("utf-8")
     print(text)
     return yaml.safe_load(text)
 
 
 class Henge(object):
-    def __init__(self, database, schemas, schemas_str=[], henges=None, 
-        checksum_function=md5):
+    def __init__(
+        self, database, schemas, schemas_str=[], henges=None, checksum_function=md5
+    ):
         """
         A user interface to insert and retrieve decomposable recursive unique
         identifiers (DRUIDs).
@@ -67,7 +75,7 @@ class Henge(object):
         self.checksum_function = checksum_function
         self.digest_version = "md5"
         self.flexible_digests = True
-
+        self.supports_inherent_attrs = True
 
         # TODO: Right now you can pass a file, or a URL, or some yaml directly
         # into the schemas param. I want to split that out so that at least the
@@ -83,7 +91,9 @@ class Henge(object):
         else:
             populated_schemas = []
             if isinstance(schemas, str):
-                _LOGGER.error("The schemas should be a list. Please pass a list of schemas")
+                _LOGGER.error(
+                    "The schemas should be a list. Please pass a list of schemas"
+                )
                 schemas = [schemas]
             for schema_value in schemas:
                 if isinstance(schema_value, str):
@@ -91,8 +101,10 @@ class Henge(object):
                         populated_schemas.append(yacman.load_yaml(schema_value))
                     elif is_url(schema_value):
                         populated_schemas.append(read_url(schema_value))
-                    else :
-                        _LOGGER.error(f"Schema file not found: {schema_value}. Use schemas_str if you meant to specify a direct schema")
+                    else:
+                        _LOGGER.error(
+                            f"Schema file not found: {schema_value}. Use schemas_str if you meant to specify a direct schema"
+                        )
                         # populated_schemas.append(yaml.safe_load(schema_value))
 
             for schema_value in schemas_str:
@@ -120,206 +132,77 @@ class Henge(object):
                     self.schemas[item_type] = henge.schemas[item_type]
                     self.henges[item_type] = henge
 
-
     def retrieve(self, druid, reclimit=None, raw=False):
+        """
+        Retrieve an item given a digest
 
-        try:
+        :param str druid: The Decomposable recursive unique identifier (DRUID), or
+            digest that uniquely identifies that item to retrieve.
+        :param int reclimit: Recursion limit. Set to None for no limit (default).
+        :param bool raw: Return the value as a raw, henge-delimited string, instead
+            of processing into a mapping. Default: False.
+        """
+        try: 
             item_type = self.database[druid + ITEM_TYPE]
-        except:
-            _LOGGER.debug(f"Item type not saved in database for {druid}")
+        except KeyError:
             raise NotFoundException(druid)
             
-        # _LOGGER.debug("item_type: {}".format(item_type))
-        # _LOGGER.debug("henge_to_query: {}".format(henge_to_query))
+        digested_string = self.lookup(druid, item_type)
+        reconstructed_item = json.loads(digested_string)
 
-        schema = self.schemas[item_type] #"type" in schema and 
-        # string = druid
-        _LOGGER.debug("Got druid to retrieve: {} / item_type: {} / schema: {}".format(
-            druid, item_type, schema))
+        external_string = self.database[druid + "_external_string"]
+        if external_string != "null":
+            external_values = json.loads(external_string)
+            reconstructed_item.update(external_values)
+
+        schema = self.schemas[item_type]
 
         if schema["type"] == "array":
-            string = self.lookup(druid, item_type)
-            _LOGGER.debug("Lookup/array/Recursive: {}; Schema: {}".format(string, schema))
-            splitstr = string.split(DELIM_ITEM)
-            # if self.flexible_digests:
-            #     pass
-            #     item_name = splitstr.pop(0)
-            if isinstance(reclimit, int) and reclimit == 0:  
-                return splitstr
-            if 'henge_class' in schema['items']:
-                _LOGGER.debug("Henge classed array: {}; Schema: {}".format(string, schema))
+            if isinstance(reclimit, int) and reclimit == 0:
+                return reconstructed_item
+            if "henge_class" in schema["items"]:
+                _LOGGER.debug(
+                    "Henge classed array: {}; Schema: {}".format(
+                        digested_string, schema
+                    )
+                )
                 if isinstance(reclimit, int):
-                    reclimit = reclimit - 1                
-                return [self.retrieve(substr, reclimit) for substr in splitstr]
-            else:
-                return splitstr
+                    reclimit = reclimit - 1
+                return [self.retrieve(item, reclimit) for item in reconstructed_item]
         elif schema["type"] == "object":
-            string = self.lookup(druid, item_type)
-            attr_array = string.split(DELIM_ATTR)
-            if self.flexible_digests:
-                keys = attr_array[::2]  # evens
-                vals = attr_array[1::2]  # odds
-                item_reconstituted = dict(zip(keys,vals))    
-            else:
-                item_reconstituted = dict(zip(schema['properties'].keys(),
-                                          attr_array))
-            # I think this part needs to be removed... it's based on the
-            # previous 'recursive' for arrays, which went away...
-            # but actually these may be added in by me, so nevermind.
-            if 'recursive' in schema:
+            if "recursive" in schema:
                 if isinstance(reclimit, int) and reclimit == 0:
-                    _LOGGER.debug("Lookup/obj/Recursive: {}; Schema: {}".format(string, schema))
-                    return item_reconstituted
+                    _LOGGER.debug(
+                        "Lookup/obj/Recursive: {}; Schema: {}".format(
+                            digested_string, schema
+                        )
+                    )
+                    return reconstructed_item
                 else:
                     if isinstance(reclimit, int):
                         reclimit = reclimit - 1
-                    for recursive_attr in schema['recursive']:                    
-                        if recursive_attr in item_reconstituted \
-                                and item_reconstituted[recursive_attr] != "":
-                            item_reconstituted[recursive_attr] = self.retrieve(
-                                item_reconstituted[recursive_attr],
-                                reclimit,
-                                raw)                
-            return item_reconstituted
-        else: # It must be a primitive type
-            # but it could be a primitive (string) that represents something to lookup,
-            # or something not-to-lookup (or already looked up)
-            _LOGGER.debug("Lookup/prim: {}; Schema: {}".format(druid, schema))
-            # return string
-            if 'henge_class' in schema and self.schemas[schema['henge_class']]['type'] in ['object', 'array']:
-                if isinstance(reclimit, int) and reclimit == 0:
-                    _LOGGER.debug("Lookup/prim/Recursive-skip: {}; Schema: {}".format(string, schema))
-                    string = self.lookup(druid, item_type)
-                    return string
-                else:
-                    if isinstance(reclimit, int):
-                        reclimit = reclimit - 1
-                    _LOGGER.debug("Lookup/prim/Recursive: {}; Schema: {}".format(druid, schema))
-                    return self.retrieve(druid, reclimit, raw)
-            else:
-                string = self.lookup(druid, item_type)
-                _LOGGER.debug("Lookup/prim/Non-recursive: {}; Schema: {}".format(string, schema))
-                return string #self.retrieve(string, reclimit, raw)      
-
-        # try:
-        #     string = henge_to_query.database[druid]
-        # except KeyError:
-        #     raise NotFoundException(druid)
-
-        # return reconstruct_item(string, schema, reclimit)
-
+                    for recursive_attr in schema["recursive"]:
+                        if (
+                            recursive_attr in reconstructed_item
+                            and reconstructed_item[recursive_attr] != ""
+                        ):
+                            reconstructed_item[recursive_attr] = self.retrieve(
+                                reconstructed_item[recursive_attr], reclimit, raw
+                            )
+        return reconstructed_item
 
     def lookup(self, druid, item_type):
         try:
             henge_to_query = self.henges[item_type]
         except:
             _LOGGER.debug("No henges available for this item type")
-            raise NotFoundException(druid)        
-        try:
-            string = henge_to_query.database[druid]
-        except KeyError:
             raise NotFoundException(druid)
-
-        return string
-
-    def retrieve2(self, druid, reclimit=None, raw=False):
-        """
-        Retrieve an item given a digest
-
-        :param str druid: The Decomposable recursive unique identifier (DRUID), or
-            digest that uniquely identifies that item to retrieve.
-        :param int reclimit: Recursion limit. Set to None for no limit (default).
-        :param bool raw: Return the value as a raw, henge-delimited string, instead
-            of processing into a mapping. Default: False.
-        """
-        def reconstruct_item(string, schema, reclimit):
-            if "type" in schema and schema["type"] == "array":
-                _LOGGER.debug("Lookup/array/Recursive: {}; Schema: {}".format(string, schema))
-                splitstr = string.split(DELIM_ITEM)
-                # if self.flexible_digests:
-                #     pass
-                #     item_name = splitstr.pop(0)
-                if 'henge_class' in schema['items'] and schema['items']['type'] not in ["object", "array"]:
-                    _LOGGER.debug("Henge classed array: {}; Schema: {}".format(string, schema))
-                    return "ASDF"
-                    return [reconstruct_item(self.henges[item_type].database[substr], schema["items"], reclimit)
-                        for substr in splitstr]
-                else:
-                    return [reconstruct_item(substr, schema["items"], reclimit)
-                        for substr in splitstr]
-            elif schema["type"] == "object":
-                attr_array = string.split(DELIM_ATTR)
-                if self.flexible_digests:
-                    keys = attr_array[::2]  # evens
-                    vals = attr_array[1::2]  # odds
-                    item_reconstituted = dict(zip(keys,vals))    
-                else:
-                    item_reconstituted = dict(zip(schema['properties'].keys(),
-                                              attr_array))
-                # I think this part needs to be removed... it's based on the
-                # previous 'recursive' for arrays, which went away...
-                # but actually these may be added in by me, so nevermind.
-                if 'recursive' in schema:
-                    if isinstance(reclimit, int) and reclimit == 0:
-                        _LOGGER.debug("Lookup/obj/Recursive: {}; Schema: {}".format(string, schema))
-                        return item_reconstituted
-                    else:
-                        if isinstance(reclimit, int):
-                            reclimit = reclimit - 1
-                        for recursive_attr in schema['recursive']:                    
-                            if item_reconstituted[recursive_attr] \
-                                    and item_reconstituted[recursive_attr] != "":
-                                item_reconstituted[recursive_attr] = self.retrieve(
-                                    item_reconstituted[recursive_attr],
-                                    reclimit,
-                                    raw)                
-                return item_reconstituted
-            else: # it must be a primitive
-                # but it could be a primitive (string) that represents something to lookup,
-                # or something not-to-lookup (or already looked up)
-                _LOGGER.debug("Lookup/prim: {}; Schema: {}".format(string, schema))
-                # return string
-                if 'henge_class' in schema and self.schemas[schema['henge_class']]['type'] in ['object', 'array']:
-                    if isinstance(reclimit, int) and reclimit == 0:
-                        _LOGGER.debug("Lookup/prim/Recursive-skip: {}; Schema: {}".format(string, schema))
-                        return string
-                    else:
-                        if isinstance(reclimit, int):
-                            reclimit = reclimit - 1
-                        _LOGGER.debug("Lookup/prim/Recursive: {}; Schema: {}".format(string, schema))
-                        return self.retrieve(string, reclimit, raw)
-                else:
-                    _LOGGER.debug("Lookup/prim/Non-recursive: {}; Schema: {}".format(string, schema))
-                    return string
-
-        # This requires the database to have __iter__ defined...and it scrolls through
-        # not a great way, take it out! 2021-01 NS
-        # I'll instead do a try block
-        # if not druid + ITEM_TYPE in self.database:
-            # raise NotFoundException(druid)
-
-        try:
-            item_type = self.database[druid + ITEM_TYPE]
-        except:
-            _LOGGER.debug(f"Item type not saved in database for {druid}")
-            raise NotFoundException(druid)
-            
-        try:
-            henge_to_query = self.henges[item_type]
-        except:
-            _LOGGER.debug("No henges available for this item type")
-            raise NotFoundException(druid)
-        # _LOGGER.debug("item_type: {}".format(item_type))
-        # _LOGGER.debug("henge_to_query: {}".format(henge_to_query))
         try:
             string = henge_to_query.database[druid]
         except KeyError:
             raise NotFoundException(druid)
 
-        schema = self.schemas[item_type]
-        _LOGGER.debug("Got druid to retrieve: {} / item_type: {} / schema: {}".format(
-            druid, item_type, schema))
-        return reconstruct_item(string, schema, reclimit)
+        return string
 
     @property
     def item_types(self):
@@ -356,20 +239,19 @@ class Henge(object):
             fits.
         """
 
-        
         _LOGGER.debug("Insert type: {} / Item: {}".format(item_type, item))
-        
+
         if item_type not in self.schemas.keys():
-            _LOGGER.error("I don't know about items of type '{}'. "
-                          "I know of: '{}'".format(item_type,
-                                                   list(self.schemas.keys())))
+            _LOGGER.error(
+                "I don't know about items of type '{}'. "
+                "I know of: '{}'".format(item_type, list(self.schemas.keys()))
+            )
             return False
 
         schema = self.schemas[item_type]
 
-
         flat_item = item
-        if schema['type'] == 'object':
+        if schema["type"] == "object":
             flat_item = {}
             if isinstance(reclimit, int) and reclimit == 0:
                 return self._insert_flat(item, item_type)
@@ -378,56 +260,63 @@ class Henge(object):
                     reclimit = reclimit - 1
                 for prop in item:
                     if prop in schema["properties"]:
-                        _LOGGER.debug("-Prop {}; Schema: {}".format(prop, str(schema["properties"][prop])))
+                        _LOGGER.debug(
+                            "-Prop {}; Schema: {}".format(
+                                prop, str(schema["properties"][prop])
+                            )
+                        )
                         if "recursive" in schema and prop in schema["recursive"]:
                             hclass = schema["properties"][prop]["henge_class"]
                             digest = self.insert(item[prop], hclass, reclimit)
                             flat_item[prop] = digest
-                        elif schema["properties"][prop]["type"] in ['array']:
+                        elif schema["properties"][prop]["type"] in ["array"]:
                             digest = self.insert(item[prop], "array", reclimit)
                             flat_item[prop] = digest
                         else:
                             flat_item[prop] = item[prop]
-                        _LOGGER.debug("Prop: {}; Flat item: {}".format(prop, flat_item[prop]))
+                        _LOGGER.debug(
+                            "Prop: {}; Flat item: {}".format(prop, flat_item[prop])
+                        )
                     else:
+                        _LOGGER.debug(f"Prop: {prop}. Ignoring due to not in schema")
                         pass  # Ignore non-schema defined properties
 
                 # if len(flat_item) == 0:
                 #     flat_item = item
-        elif schema['type'] == 'array':
+        elif schema["type"] == "array":
             flat_item = []
-            if 'henge_class' in schema['items']:
+            if "henge_class" in schema["items"]:
                 digest = []
-                hclass = schema['items']["henge_class"]
+                hclass = schema["items"]["henge_class"]
                 if isinstance(reclimit, int) and reclimit == 0:
                     return self._insert_flat(item, item_type)
                 else:
                     if isinstance(reclimit, int):
-                        reclimit = reclimit - 1                    
-                    _LOGGER.debug("Item: {}. Pyclass: {}. hclass: {}".format(item, type(item), hclass))
+                        reclimit = reclimit - 1
+                    _LOGGER.debug(
+                        "Item: {}. Pyclass: {}. hclass: {}".format(
+                            item, type(item), hclass
+                        )
+                    )
                     for element in item:
                         digest.append(self.insert(element, hclass, reclimit))
                     flat_item = digest
             else:
                 flat_item = item
                 _LOGGER.debug("Array flat item: {}".format(flat_item))
-        else: # A primitive type with a henge class
+        else:  # A primitive type with a henge class
             _LOGGER.debug("Nice! You're using a henge-classed primitive type!")
             hclass = schema["henge_class"]
             # digest = self.insert(item, hclass)
             flat_item = item
 
-
-
-
         return self._insert_flat(flat_item, item_type)
 
-
     def _insert_flat(self, item, item_type=None, item_name=None):
         """
         Add flattened items (of a specified type) to the database.
 
-        Flattened items have removed all levels, so it's only attributes and 
+        Flattened items have removed all levels, so it's only attributes and
         strict values; no nesting allowed. Use the upstream insert function
         to insert full structured objects, which calls this function.
 
@@ -438,9 +327,10 @@ class Henge(object):
             fits.
         """
         if item_type not in self.schemas.keys():
-            _LOGGER.error("I don't know about items of type '{}'. "
-                          "I know of: '{}'".format(item_type,
-                                                   list(self.schemas.keys())))
+            _LOGGER.error(
+                "I don't know about items of type '{}'. "
+                "I know of: '{}'".format(item_type, list(self.schemas.keys()))
+            )
             return False
 
         # digest_version should be automatically appended to the item by the
@@ -449,45 +339,16 @@ class Henge(object):
         # jsonschema do this automatically?
         # also item_type ?
 
-        def safestr(item, x):
-            try:
-                return str(item[x])
-            except (ValueError, TypeError, KeyError):
-                return ""
-
-        def build_attr_string(item, schema, item_name=None):
-            if "type" in schema and schema["type"] == "array":
-                if self.flexible_digests:
-                    return DELIM_ITEM.join([build_attr_string(x, schema['items'])
-                                        for x in item])
-                else:
-                    return DELIM_ITEM.join([build_attr_string(x, schema['items'])
-                                        for x in item])
-            elif schema["type"] == "object" and 'properties' in schema:
-                if self.flexible_digests:
-                    # flexible schema
-                    keys_to_include = sorted([x for x in item.keys() if x in list(schema['properties'].keys())])
-                    return DELIM_ATTR.join([DELIM_ATTR.join([k, safestr(item, k)]) for k in keys_to_include])
-
-                else:
-                    # fixed schema
-                    return DELIM_ATTR.join([safestr(item, x) for x in
-                                        list(schema['properties'].keys())])
-            else: #assume it's a primitive
-                if self.flexible_digests:
-                    return item
-                    attr_string = DELIM_ATTR.join([item_name, item])
-                    return attr_string
-                else:
-                    return item
-
         valid_schema = self.schemas[item_type]
         # Add defaults here ?
-        try: 
+        try:
             jsonschema.validate(item, valid_schema)
         except jsonschema.ValidationError as e:
-            _LOGGER.error("Not valid data. Item type: {}. Attempting to insert item: {}".
-                format(item_type, item))
+            _LOGGER.error(
+                "Not valid data. Item type: {}. Attempting to insert item: {}".format(
+                    item_type, item
+                )
+            )
             print(e)
 
             if isinstance(item, str):
@@ -495,7 +356,9 @@ class Henge(object):
                 try:
                     existing_item_type = henge_to_query.database[item + ITEM_TYPE]
                 except KeyError:
-                    _LOGGER.error("If you're trying to insert an item with druids, the sub-items must exist in the database.")
+                    _LOGGER.error(
+                        "If you're trying to insert an item with druids, the sub-items must exist in the database."
+                    )
                     # return None
                 try:
                     existing_item = henge_to_query.database[item]
@@ -503,23 +366,32 @@ class Henge(object):
                     _LOGGER.error("That item wasn't in the database.")
 
                 # if (item_type == existing_item_type):
-                    # _LOGGER.info("But wait!!! That's already here, and it's great! I'll return that!")
+                # _LOGGER.info("But wait!!! That's already here, and it's great! I'll return that!")
                 return item
 
             raise e
             return None
-            
-        attr_string = build_attr_string(item, valid_schema)
+
+        _LOGGER.debug(f"item to insert: {item}")
+        item_inherent_split = select_inherent_properties(item, valid_schema)
+        attr_string = canonical_str(item_inherent_split["inherent"])
+        external_string = canonical_str(item_inherent_split["external"])
 
         _LOGGER.debug(f"String to digest: {attr_string}")
+        _LOGGER.debug(f"External string: {external_string}")
         druid = self.checksum_function(attr_string)
-        self._henge_insert(druid, attr_string, item_type)
+        self._henge_insert(druid, attr_string, item_type, external_string)
 
-        _LOGGER.debug("Inserted flat item. Digest: {} / Type: {} / Item: {}".format(
-            druid, item_type, item))
+        _LOGGER.debug(
+            "Inserted flat item. Digest: {} / Type: {} / Item: {}".format(
+                druid, item_type, item
+            )
+        )
         return druid
 
-    def _henge_insert(self, druid, string, item_type, digest_version=None):
+    def _henge_insert(
+        self, druid, string, item_type, external_string, digest_version=None
+    ):
         """
         Inserts an item into the database, with henge-metadata slots for item
         type and digest version.
@@ -540,6 +412,7 @@ class Henge(object):
             henge_to_query.database[druid] = string
             henge_to_query.database[druid + ITEM_TYPE] = item_type
             henge_to_query.database[druid + "_digest_version"] = digest_version
+            henge_to_query.database[druid + "_external_string"] = external_string
 
             if henge_to_query != self:
                 self.database[druid + ITEM_TYPE] = item_type
@@ -551,13 +424,16 @@ class Henge(object):
         """
         Remove all items from this database.
         """
-        for k, v in self.database.items():
-            try:
-                del self.database[k]
-                del self.database[k + ITEM_TYPE]
-                del self.database[k + "_digest_version"]
-            except (KeyError, AttributeError):
-                pass
+        try:
+            for k, v in self.database.items():
+                try:
+                    del self.database[k]
+                    del self.database[k + ITEM_TYPE]
+                    del self.database[k + "_digest_version"]
+                except (KeyError, AttributeError):
+                    pass
+        except AttributeError as e:
+            _LOGGER.warn(f"Error trying to iterate over database items: {e}")
 
     def show(self):
         """
@@ -566,7 +442,6 @@ class Henge(object):
         for k, v in self.database.items():
             print(k, v)
 
-
     def __repr__(self):
         repr = "Henge object. Item types: " + ",".join(self.item_types)
         return repr
@@ -578,68 +453,74 @@ def split_schema(schema, name=None):
     """
     slist = {}
     # base case
-    if schema['type'] not in ['object', 'array']:
+    if schema["type"] not in ["object", "array"]:
         _LOGGER.debug(schema)
         if name:
             slist[name] = schema
-        elif 'henge_class' in schema:
-            slist[schema['henge_class']] = schema
+        elif "henge_class" in schema:
+            slist[schema["henge_class"]] = schema
         _LOGGER.debug("Returning slist: {}".format(str(slist)))
         return slist
-    elif schema['type'] == 'object':
+    elif schema["type"] == "object":
         recursive_properties = []
-        if 'henge_class' in schema:
+        if "henge_class" in schema:
             schema_copy = copy.deepcopy(schema)
-            _LOGGER.debug("adding " + str(schema_copy['henge_class']))
-            henge_class = schema_copy['henge_class']
+            _LOGGER.debug("adding " + str(schema_copy["henge_class"]))
+            henge_class = schema_copy["henge_class"]
             # del schema_copy['henge_class']
-            for p in schema_copy['properties']:
+            for p in schema_copy["properties"]:
                 hclass = None
-                if 'henge_class' in schema_copy['properties'][p]:
-                    hclass = schema_copy['properties'][p]['henge_class']
+                if "henge_class" in schema_copy["properties"][p]:
+                    hclass = schema_copy["properties"][p]["henge_class"]
                     recursive_properties.append(p)
-                if schema_copy['properties'][p]['type'] in ['object']:
+                if schema_copy["properties"][p]["type"] in ["object"]:
                     # recursive_properties.append(p)
-                    schema_copy['properties'][p] = {'type': "string"}
+                    schema_copy["properties"][p] = {"type": "string"}
                     if hclass:
-                        schema_copy['properties'][p]['henge_class'] = hclass
-                if schema_copy['properties'][p]['type'] in ["array"]:
+                        schema_copy["properties"][p]["henge_class"] = hclass
+                if schema_copy["properties"][p]["type"] in ["array"]:
                     # recursive_properties.append(p)
-                    schema_copy['properties'][p] = {'type': "string"}
+                    if schema_copy["properties"][p]["items"]["type"] == "integer":
+                        schema_copy["properties"][p] = {"type": "string"}
+                    else:
+                        schema_copy["properties"][p] = {"type": "string"}
                     if hclass:
-                        schema_copy['properties'][p]['henge_class'] = hclass
+                        schema_copy["properties"][p]["henge_class"] = hclass
                     else:
-                        schema_copy['properties'][p]['henge_class'] = "array"
+                        schema_copy["properties"][p]["henge_class"] = "strarray"
                     # schema_copy['properties'][p]['type'] = "string"
             # del schema_copy['properties']
-            _LOGGER.debug("Adding recursive properties: {}".format(recursive_properties))
-            schema_copy['recursive'] = recursive_properties
+            _LOGGER.debug(
+                "Adding recursive properties: {}".format(recursive_properties)
+            )
+            schema_copy["recursive"] = recursive_properties
             slist[henge_class] = schema_copy
 
-        for p in schema['properties']:
+        for p in schema["properties"]:
             # if schema['properties'][p]['type'] in ['object', 'array']:
-            #     recursive_properties.append(p)            
-            schema_sub = schema['properties'][p]
+            #     recursive_properties.append(p)
+            schema_sub = schema["properties"][p]
             _LOGGER.debug("checking property:" + p)
-            slist.update(split_schema(schema['properties'][p]))
-    elif schema['type'] == 'array':
+            slist.update(split_schema(schema["properties"][p]))
+    elif schema["type"] == "array":
         _LOGGER.debug("found array")
         _LOGGER.debug(schema)
-        if 'henge_class' in schema:
+        if "henge_class" in schema:
             schema_copy = copy.deepcopy(schema)
-            _LOGGER.debug("adding " + str(schema['henge_class']))
-            henge_class = schema_copy['henge_class']
+            _LOGGER.debug("adding " + str(schema["henge_class"]))
+            henge_class = schema_copy["henge_class"]
             # del schema_copy['henge_class']
-            schema_copy['items'] = {'type': "string"}
-            if 'recursive' in schema_copy and schema_copy['recursive']:
-                schema_copy['items']['recursive'] = True
-            if 'henge_class' in schema['items']:
-                schema_copy['items']['henge_class'] = schema['items']['henge_class']
+            if schema_copy["items"]["type"] != "integer":
+                schema_copy["items"] = {"type": "string"}
+            if "recursive" in schema_copy and schema_copy["recursive"]:
+                schema_copy["items"]["recursive"] = True
+            if "henge_class" in schema["items"]:
+                schema_copy["items"]["henge_class"] = schema["items"]["henge_class"]
             # schema_copy['items']['type'] = "string"
             # if 'properties' in schema_copy['items']:
             #     del schema_copy['items']['properties']
             slist[henge_class] = schema_copy
-            schema_sub = schema['items']
+            schema_sub = schema["items"]
             slist.update(split_schema(schema_sub))
         else:
             _LOGGER.debug("Classless array")
@@ -650,24 +531,46 @@ def split_schema(schema, name=None):
     return slist
 
 
+def canonical_str(item: dict) -> str:
+    """Convert a dict into a canonical string representation"""
+    return json.dumps(
+        item, separators=(",", ":"), ensure_ascii=False, allow_nan=False, sort_keys=True
+    )
+
+
+def select_inherent_properties(item: dict, schema: dict) -> dict:
+    if schema["type"] == "object":
+        item_inherent = {}
+        if "inherent" in schema and schema["inherent"]:
+            for k in schema["inherent"]:
+                item_inherent[k] = item[k]
+                del item[k]
+            return {"inherent": item_inherent, "external": item}
+        else:
+            return {"inherent": item, "external": None}
+    else:
+        return {"inherent": item, "external": None}
+
+
 def is_schema_recursive(schema):
     """
     Determine if a given schema has elements that need to recurse
     """
     # return 'recursive' in schema # old way
     is_recursive = False
-    if schema['type'] == "object":
-        for prop in schema['properties']:
-            if schema['properties']['prop']['type'] in ['object', 'array']:
+    if schema["type"] == "object":
+        for prop in schema["properties"]:
+            if schema["properties"]["prop"]["type"] in ["object", "array"]:
                 return True
-    if schema['type'] == "array":        
-        if schema['items']['type'] in ['object', 'array']:
+    if schema["type"] == "array":
+        if schema["items"]["type"] in ["object", "array"]:
             return True
     return False
 
 
-def connect_mongo(host='0.0.0.0', port=27017, database='henge_dict',
-                  collection='store'):
+def connect_mongo(
+    host="0.0.0.0", port=27017, database="henge_dict", collection="store"
+):
     """
     Connect to MongoDB and return the MongoDB-backed dict object
 
@@ -682,64 +585,20 @@ def connect_mongo(host='0.0.0.0', port=27017, database='henge_dict',
     """
     from importlib import import_module
     from inspect import stack
+
     for lib in LIBS_BY_BACKEND["mongo"]:
         try:
             globals()[lib] = import_module(lib)
         except ImportError:
             raise ImportError(
                 "Requirements not met. Package '{}' is required to setup "
-                "MongoDB connection. Install the package and call '{}' again.".
-                    format(lib, stack()[0][3]))
-    pymongo.Connection = lambda host, port, **kwargs: \
-        pymongo.MongoClient(host=host, port=port)
-    return mongodict.MongoDict(host=host, port=port, database=database,
-                               collection=collection)
-
-
-def build_argparser():
-    # TODO: add cli hooks for ^
-    """
-    Builds argument parser.
-
-    :return argparse.ArgumentParser
-    """
-    banner = "%(prog)s - Keeper of druids: " \
-             "a python interface to Decomposable Recursive UIDs"
-    additional_description = "\n..."
-    parser = VersionInHelpParser(version=__version__, description=banner,
-                                 epilog=additional_description)
-
-    parser.add_argument(
-            "-V", "--version",
-            action="version",
-            version="%(prog)s {v}".format(v=__version__))
-
-    parser.add_argument(
-            "-i", "--input", required=True,
-            help="File path to input file.")
-
-    parser.add_argument(
-            "-p", "--parameter", type=int, default=0,
-            help="Some parameter.")
-
-    return parser
-
-
-def main():
-    """ Primary workflow """
-
-    parser = logmuse.add_logging_options(build_argparser())
-    args = parser.parse_args()
-    global _LOGGER
-    _LOGGER = logmuse.logger_via_cli(args, make_root=True)
-
-    msg = "Input: {input}; Parameter: {parameter}"
-    _LOGGER.info(msg.format(input=args.input, parameter=args.parameter))
-
-
-if __name__ == '__main__':
-    try:
-        sys.exit(main())
-    except KeyboardInterrupt:
-        _LOGGER.error("Program canceled by user!")
-        sys.exit(1)
+                "MongoDB connection. Install the package and call '{}' again.".format(
+                    lib, stack()[0][3]
+                )
+            )
+    pymongo.Connection = lambda host, port, **kwargs: pymongo.MongoClient(
+        host=host, port=port
+    )
+    return mongodict.MongoDict(
+        host=host, port=port, database=database, collection=collection
+    )

{henge-0.1.1.dist-info → henge-0.2.1.dist-info}/METADATA

@@ -1,20 +1,20 @@
 Metadata-Version: 2.1
 Name: henge
-Version: 0.1.1
+Version: 0.2.1
 Summary: Storage and retrieval of object-derived, decomposable recursive unique identifiers.
 Home-page: https://databio.org
 Author: Nathan Sheffield
 Author-email: nathan@code.databio.org
 License: BSD2
-Platform: UNKNOWN
 Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: BSD License
-Classifier: Programming Language :: Python :: 2.7
-Classifier: Programming Language :: Python :: 3.5
-Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
 Classifier: Topic :: System :: Distributed Computing
 Description-Content-Type: text/markdown
-Requires-Dist: logmuse
+License-File: LICENSE.txt
 Requires-Dist: jsonschema
 Requires-Dist: ubiquerg (>=0.5.2)
 Requires-Dist: yacman (>=0.6.7)
@@ -26,5 +26,3 @@ Requires-Dist: yacman (>=0.6.7)
 Henge is a Python package that builds backends for generic decomposable recursive unique identifiers (or, *DRUIDs*). It is intended to be used as a building block for sequence collections (see the [seqcol package](https://github.com/databio/seqcol)), and also for other data types that need content-derived identifiers.
 
 Documentation at [http://henge.databio.org](http://henge.databio.org).
-
-

henge-0.2.1.dist-info/RECORD

@@ -0,0 +1,11 @@
+henge/__init__.py,sha256=Su4ZQre-GS24y-LjiAIf57GYovLTww0hkk2fDxzMt_g,289
+henge/_version.py,sha256=Zn1KFblwuFHiDRdRAiRnDBRkbPttWh44jKa5zG2ov0E,22
+henge/const.py,sha256=0t3EgqdjmKBd-zu5L8AJnGoWv0T3sAtvXf-3b62Dd-Y,194
+henge/deprecated.py,sha256=C8eINR2gWCiNaN2b3gbFYn8jfJ0ftJm8a_fIgVVVzXc,11248
+henge/henge.py,sha256=mvAcXefMy5r8XcgRe9XXPH5uuXdyL9-sXRYWLNp_fos,23454
+henge-0.2.1.dist-info/LICENSE.txt,sha256=oB6ZGDa4kcznznJKJsLLFFcOZyi8Y6e2Jv0rJozgp-I,1269
+henge-0.2.1.dist-info/METADATA,sha256=zW6QW4rlOKOpkymjUGkvlMRhbdXZQU_7YvTx3WM2ap8,1270
+henge-0.2.1.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
+henge-0.2.1.dist-info/entry_points.txt,sha256=c2OKgrH1a5Cx2osbUFSe9NFK8CbN82lPPsi4wry77_M,61
+henge-0.2.1.dist-info/top_level.txt,sha256=QyovlLuKhhKP1r8bMVmxLdke9F6PZFIN7VlkzvB0xIQ,6
+henge-0.2.1.dist-info/RECORD,,

{henge-0.1.1.dist-info → henge-0.2.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.36.2)
+Generator: bdist_wheel (0.38.4)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{henge-0.1.1.dist-info → henge-0.2.1.dist-info}/entry_points.txt

@@ -1,3 +1,2 @@
 [console_scripts]
 packagename = packagename.packagename:main
-

henge-0.1.1.dist-info/RECORD

@@ -1,10 +0,0 @@
-henge/__init__.py,sha256=mgUuUeNMFlWAwfiyjj1YFSVjGzAoyKB_lr56cYygiA0,272
-henge/_version.py,sha256=rnObPjuBcEStqSO0S6gsdS_ot8ITOQjVj_-P1LUUYpg,22
-henge/const.py,sha256=BW_CMh8eU7QR8umjZA3Th3uNHZNi7jmbDp9J_uHg5to,193
-henge/henge.py,sha256=y0oQIM_icunEiSN4_9hNw2HNAYki0wm4ehV0dY9dFYM,31385
-henge-0.1.1.dist-info/LICENSE.txt,sha256=oB6ZGDa4kcznznJKJsLLFFcOZyi8Y6e2Jv0rJozgp-I,1269
-henge-0.1.1.dist-info/METADATA,sha256=ljP5d6hCmcOCnObmaXLdzn3lQV8MRPjIfjIO2lRntws,1236
-henge-0.1.1.dist-info/WHEEL,sha256=OqRkF0eY5GHssMorFjlbTIq072vpHpF60fIQA6lS9xA,92
-henge-0.1.1.dist-info/entry_points.txt,sha256=XzpBzx_VAbjQiN-QErDU5al7HZhTtI1nZKvbz17guQQ,62
-henge-0.1.1.dist-info/top_level.txt,sha256=QyovlLuKhhKP1r8bMVmxLdke9F6PZFIN7VlkzvB0xIQ,6
-henge-0.1.1.dist-info/RECORD,,