henge 0.2.2__py3-none-any.whl → 0.2.3__py3-none-any.whl

henge/__init__.py

@@ -9,5 +9,5 @@ __all__ = __classes__ + [
     "split_schema",
     "NotFoundException",
     "canonical_str",
-    "sha512t24u_digest"
+    "sha512t24u_digest",
 ]

henge/_version.py

@@ -1 +1 @@
-__version__ = "0.2.1"
+__version__ = "0.2.3"

henge/henge.py

@@ -1,4 +1,4 @@
-""" An interface to a database back-end for DRUIDs """
+"""An interface to a database back-end for DRUIDs"""
 
 import base64
 import copy
@@ -30,11 +30,12 @@ class NotFoundException(Exception):
 
 
 def sha512t24u_digest(seq: str, offset: int = 24) -> str:
-    """ GA4GH digest function """
+    """GA4GH digest function"""
     digest = hashlib.sha512(seq.encode()).digest()
     tdigest_b64us = base64.urlsafe_b64encode(digest[:offset])
     return tdigest_b64us.decode("ascii")
 
+
 def md5(seq):
     return hashlib.md5(seq.encode()).hexdigest()
 
@@ -56,7 +57,6 @@ def read_url(url):
         raise e
     data = response.read()  # a `bytes` object
     text = data.decode("utf-8")
-    print(text)
     return yaml.safe_load(text)
 
 
@@ -250,8 +250,9 @@ class Henge(object):
 
         if item_type not in self.schemas.keys():
             _LOGGER.error(
-                "I don't know about items of type '{}'. "
-                "I know of: '{}'".format(item_type, list(self.schemas.keys()))
+                "I don't know about items of type '{}'. I know of: '{}'".format(
+                    item_type, list(self.schemas.keys())
+                )
             )
             return False
 
@@ -335,8 +336,9 @@ class Henge(object):
         """
         if item_type not in self.schemas.keys():
             _LOGGER.error(
-                "I don't know about items of type '{}'. "
-                "I know of: '{}'".format(item_type, list(self.schemas.keys()))
+                "I don't know about items of type '{}'. I know of: '{}'".format(
+                    item_type, list(self.schemas.keys())
+                )
             )
             return False
 
@@ -449,6 +451,9 @@ class Henge(object):
         for k, v in self.database.items():
             print(k, v)
 
+    def __len__(self):
+        return len(self.database)
+
     def list(self, limit=1000, offset=0):
         """
         List all items in the database.
@@ -457,7 +462,7 @@ class Henge(object):
             "count": len(self.database),
             "limit": limit,
             "offset": offset,
-            "items": list(self.database.keys())[offset:(offset+limit)],
+            "items": list(self.database.keys())[offset : (offset + limit)],
         }
 
     def __repr__(self):

henge/scconf.py

@@ -0,0 +1,359 @@
+import logging
+import os
+import psycopg2
+
+from collections.abc import Mapping
+from psycopg2 import OperationalError, sql
+from psycopg2.errors import UniqueViolation
+
+_LOGGER = logging.getLogger(__name__)
+
+# Use like:
+# pgdb = RDBDict(...)       # Open connection
+# pgdb["key"] = "value"     # Insert item
+# pgdb["key"]               # Retrieve item
+# pgdb.close()              # Close connection
+
+
+# This was originally written in seqcolapi.
+# I am moving it here in 2025, because the whole point was to enable
+# interesting database back-ends to have dict-style key-value pair
+# mechanisms, which was enabling henge to use these various backends
+# to back arbitrary databases.
+# with the move to sqlmodel, I abandoned the henge backend approach,
+# so intermediates are no longer important for seqcol.
+
+# they could become relevant for other henge use cases, so they
+# fit better here now.
+
+
+def getenv(varname):
+    """Simple wrapper to make the Exception more informative for missing env var"""
+    try:
+        return os.environ[varname]
+    except KeyError:
+        raise Exception(f"Environment variable {varname} not set.")
+
+
+import pipestat
+
+
+class PipestatMapping(pipestat.PipestatManager):
+    """A wrapper class to allow using a PipestatManager as a dict-like object."""
+
+    def __getitem__(self, key):
+        # This little hack makes this work with `in`;
+        # e.g.: for x in rdbdict, which is now disabled, instead of infinite.
+        if isinstance(key, int):
+            raise IndexError
+        return self.retrieve(key)
+
+    def __setitem__(self, key, value):
+        return self.insert({key: value})
+
+    def __len__(self):
+        return self.count_records()
+
+    def _next_page(self):
+        self._buf["page_index"] += 1
+        limit = self._buf["page_size"]
+        offset = self._buf["page_index"] * limit
+        self._buf["keys"] = self.get_records(limit, offset)
+        return self._buf["keys"][0]
+
+    def __iter__(self):
+        _LOGGER.debug("Iterating...")
+        self._buf = {  # buffered iterator
+            "current_view_index": 0,
+            "len": len(self),
+            "page_size": 100,
+            "page_index": -1,
+            "keys": self._next_page(),
+        }
+        return self
+
+    def __next__(self):
+        if self._buf["current_view_index"] > self._buf["len"]:
+            raise StopIteration
+
+        idx = (
+            self._buf["current_view_index"]
+            - self._buf["page_index"] * self._buf["page_size"]
+        )
+        if idx <= self._buf["page_size"]:
+            self._buf["current_view_index"] += 1
+            return self._buf["keys"][idx - 1]
+        else:  # current index is beyond current page, but not beyond total
+            return self._next_page()
+
+
+class RDBDict(Mapping):
+    """
+    A Relational DataBase Dict.
+
+    Simple database connection manager object that allows us to use a
+    PostgresQL database as a simple key-value store to back Python
+    dict-style access to database items.
+    """
+
+    def __init__(
+        self,
+        db_name: str = None,
+        db_user: str = None,
+        db_password: str = None,
+        db_host: str = None,
+        db_port: str = None,
+        db_table: str = None,
+    ):
+        self.connection = None
+        self.db_name = db_name or getenv("POSTGRES_DB")
+        self.db_user = db_user or getenv("POSTGRES_USER")
+        self.db_host = db_host or os.environ.get("POSTGRES_HOST") or "localhost"
+        self.db_port = db_port or os.environ.get("POSTGRES_PORT") or "5432"
+        self.db_table = db_table or os.environ.get("POSTGRES_TABLE") or "seqcol"
+        db_password = db_password or getenv("POSTGRES_PASSWORD")
+
+        try:
+            self.connection = self.create_connection(
+                self.db_name, self.db_user, db_password, self.db_host, self.db_port
+            )
+            if not self.connection:
+                raise Exception("Connection failed")
+        except Exception as e:
+            _LOGGER.info(f"{self}")
+            raise e
+        _LOGGER.info(self.connection)
+        self.connection.autocommit = True
+
+    def __repr__(self):
+        return (
+            "RDBD object\n"
+            + "db_table: {}\n".format(self.db_table)
+            + "db_name: {}\n".format(self.db_name)
+            + "db_user: {}\n".format(self.db_user)
+            + "db_host: {}\n".format(self.db_host)
+            + "db_port: {}\n".format(self.db_port)
+        )
+
+    def init_table(self):
+        # Wrap statements to prevent SQL injection attacks
+        stmt = sql.SQL(
+            """
+            CREATE TABLE IF NOT EXISTS {table}(
+            key TEXT PRIMARY KEY, 
+            value TEXT);
+        """
+        ).format(table=sql.Identifier(self.db_table))
+        return self.execute_query(stmt, params=None)
+
+    def insert(self, key, value):
+        stmt = sql.SQL(
+            """
+            INSERT INTO {table}(key, value)
+            VALUES (%(key)s, %(value)s);
+        """
+        ).format(table=sql.Identifier(self.db_table))
+        params = {"key": key, "value": value}
+        return self.execute_query(stmt, params)
+
+    def update(self, key, value):
+        stmt = sql.SQL(
+            """
+            UPDATE {table} SET value=%(value)s WHERE key=%(key)s
+        """
+        ).format(table=sql.Identifier(self.db_table))
+        params = {"key": key, "value": value}
+        return self.execute_query(stmt, params)
+
+    def __getitem__(self, key):
+        # This little hack makes this work with `in`;
+        # e.g.: for x in rdbdict, which is now disabled, instead of infinite.
+        if isinstance(key, int):
+            raise IndexError
+        stmt = sql.SQL(
+            """
+            SELECT value FROM {table} WHERE key=%(key)s
+        """
+        ).format(table=sql.Identifier(self.db_table))
+        params = {"key": key}
+        res = self.execute_read_query(stmt, params)
+        if not res:
+            _LOGGER.info("Not found: {}".format(key))
+        return res
+
+    def __setitem__(self, key, value):
+        try:
+            return self.insert(key, value)
+        except UniqueViolation as e:
+            _LOGGER.info("Updating existing value for {}".format(key))
+            return self.update(key, value)
+
+    def __delitem__(self, key):
+        stmt = sql.SQL(
+            """
+            DELETE FROM {table} WHERE key=%(key)s
+        """
+        ).format(table=sql.Identifier(self.db_table))
+        params = {"key": key}
+        res = self.execute_query(stmt, params)
+        return res
+
+    def create_connection(self, db_name, db_user, db_password, db_host, db_port):
+        connection = None
+        try:
+            connection = psycopg2.connect(
+                database=db_name,
+                user=db_user,
+                password=db_password,
+                host=db_host,
+                port=db_port,
+            )
+            _LOGGER.info("Connection to PostgreSQL DB successful")
+        except OperationalError as e:
+            _LOGGER.info("Error: {e}".format(e=str(e)))
+        return connection
+
+    def execute_read_query(self, query, params=None):
+        cursor = self.connection.cursor()
+        result = None
+        try:
+            cursor.execute(query, params)
+            result = cursor.fetchone()
+            if result:
+                return result[0]
+            else:
+                _LOGGER.debug(f"Query: {query}")
+                _LOGGER.debug(f"Result: {result}")
+                return None
+        except OperationalError as e:
+            _LOGGER.info("Error: {e}".format(e=str(e)))
+            raise
+        except TypeError as e:
+            _LOGGER.info("TypeError: {e}, item: {q}".format(e=str(e), q=query))
+            raise
+
+    def execute_multi_query(self, query, params=None):
+        cursor = self.connection.cursor()
+        result = None
+        try:
+            cursor.execute(query, params)
+            result = cursor.fetchall()
+            return result
+        except OperationalError as e:
+            _LOGGER.info("Error: {e}".format(e=str(e)))
+            raise
+        except TypeError as e:
+            _LOGGER.info("TypeError: {e}, item: {q}".format(e=str(e), q=query))
+            raise
+
+    def execute_query(self, query, params=None):
+        cursor = self.connection.cursor()
+        try:
+            return cursor.execute(query, params)
+            _LOGGER.info("Query executed successfully")
+        except OperationalError as e:
+            _LOGGER.info("Error: {e}".format(e=str(e)))
+
+    def close(self):
+        _LOGGER.info("Closing connection")
+        return self.connection.close()
+
+    def __del__(self):
+        if self.connection:
+            self.close()
+
+    def __len__(self):
+        stmt = sql.SQL(
+            """
+            SELECT COUNT(*) FROM {table}
+        """
+        ).format(table=sql.Identifier(self.db_table))
+        _LOGGER.debug(stmt)
+        res = self.execute_read_query(stmt)
+        return res
+
+    def get_paged_keys(self, limit=None, offset=None):
+        stmt = sql.SQL("SELECT key FROM {table}").format(
+            table=sql.Identifier(self.db_table)
+        )
+        params = {}
+        if limit is not None:
+            stmt = sql.SQL("{} LIMIT %(limit)s").format(stmt)
+            params["limit"] = limit
+        if offset is not None:
+            stmt = sql.SQL("{} OFFSET %(offset)s").format(stmt)
+            params["offset"] = offset
+        res = self.execute_multi_query(stmt, params if params else None)
+        return res
+
+    def _next_page(self):
+        self._buf["page_index"] += 1
+        limit = self._buf["page_size"]
+        offset = self._buf["page_index"] * limit
+        self._buf["keys"] = self.get_paged_keys(limit, offset)
+        return self._buf["keys"][0]
+
+    def __iter__(self):
+        _LOGGER.debug("Iterating...")
+        self._buf = {  # buffered iterator
+            "current_view_index": 0,
+            "len": len(self),
+            "page_size": 10,
+            "page_index": 0,
+            "keys": self.get_paged_keys(10, 0),
+        }
+        return self
+
+    def __next__(self):
+        if self._buf["current_view_index"] > self._buf["len"]:
+            raise StopIteration
+
+        idx = (
+            self._buf["current_view_index"]
+            - self._buf["page_index"] * self._buf["page_size"]
+        )
+        if idx <= self._buf["page_size"]:
+            self._buf["current_view_index"] += 1
+            return self._buf["keys"][idx - 1]
+        else:  # current index is beyond current page, but not beyond total
+            return self._next_page()
+
+    # Old, non-paged iterator:
+    # def __iter__(self):
+    #     self._current_idx = 0
+    #     return self
+
+    # def __next__(self):
+    #     stmt = sql.SQL(
+    #         """
+    #         SELECT key,value FROM {table} LIMIT 1 OFFSET %(idx)s
+    #     """
+    #     ).format(table=sql.Identifier(self.db_table))
+    #     res = self.execute_read_query(stmt, {"idx": self._current_idx})
+    #     self._current_idx += 1
+    #     if not res:
+    #         _LOGGER.info("Not found: {}".format(self._current_idx))
+    #         raise StopIteration
+    #     return res
+
+
+# We don't need the full SeqColHenge,
+# which also has loading capability, and requires pyfaidx, which requires
+# biopython, which requires numpy, which is huge and can't compile the in
+# default fastapi container.
+# So, I had written the below class which provides retrieve only.
+# HOWEVER, switching from alpine to slim allows install of numpy;
+# This inflates the container size from 262Mb to 350Mb; perhaps that's worth paying.
+# So I can avoid duplicating this and just use the full SeqColHenge from seqcol
+# class SeqColHenge(refget.RefGetClient):
+#     def retrieve(self, druid, reclimit=None, raw=False):
+#         try:
+#             return super(SeqColHenge, self).retrieve(druid, reclimit, raw)
+#         except henge.NotFoundException as e:
+#             _LOGGER.debug(e)
+#             try:
+#                 return self.refget(druid)
+#             except Exception as e:
+#                 _LOGGER.debug(e)
+#                 raise e
+#                 return henge.NotFoundException("{} not found in database, or in refget.".format(druid))

henge-0.2.3.dist-info/METADATA

@@ -0,0 +1,132 @@
+Metadata-Version: 2.4
+Name: henge
+Version: 0.2.3
+Summary: Storage and retrieval of object-derived, decomposable recursive unique identifiers.
+Home-page: https://databio.org
+Author: Nathan Sheffield
+Author-email: nathan@code.databio.org
+License: BSD2
+Classifier: Development Status :: 4 - Beta
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: System :: Distributed Computing
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE.txt
+Requires-Dist: jsonschema
+Requires-Dist: ubiquerg>=0.5.2
+Requires-Dist: yacman>=0.6.7
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Henge
+
+Henge is a Python package for building data storage and retrieval interfaces for arbitrary data. Henge is based on the idea of **decomposable recursive unique identifiers (DRUIDs)**, which are hash-based unique identifiers for data derived from the data itself. For arbitrary data with any structure, Henge can mint unique DRUIDs to identify data, store the data in a key-value database of your choice, and provide lookup functions to retrieve the data in its original structure using its DRUID identifier.
+
+Henge was intended as a building block for [sequence collections](https://github.com/refgenie/seqcol), but is generic enough to use for any data type that needs content-derived identifiers with database lookup capability.
+
+## Install
+
+```
+pip install henge
+```
+
+## Quick Start
+
+Create a Henge object by providing a database and a data schema. The database can be a Python dict or backed by persistent storage. Data schemas are [JSON-schema](https://json-schema.org/) descriptions of data types, and can be hierarchical.
+
+```python
+import henge
+
+schemas = ["path/to/json_schema.yaml"]
+h = henge.Henge(database={}, schemas=schemas)
+```
+
+Insert items into the henge. Upon insert, henge returns the DRUID (digest/checksum/unique identifier) for your object:
+
+```python
+druid = h.insert({"name": "Pat", "age": 38}, item_type="person")
+```
+
+Retrieve the original object using the DRUID:
+
+```python
+h.retrieve(druid)
+# {'age': '38', 'name': 'Pat'}
+```
+
+## Tutorial
+
+For a comprehensive walkthrough covering basic types, arrays, nested objects, and advanced features, see the [tutorial notebook](docs/tutorial.ipynb).
+
+## What are DRUIDs?
+
+DRUIDs are a special type of unique identifiers with two powerful properties:
+
+- **Decomposable**: Identifiers in henge automatically retrieve structured data (tuples, arrays, objects). The structure is defined by a JSON schema, so henge can be used as a back-end for arbitrary data types.
+
+- **Recursive**: Individual elements retrieved by henge can be tagged as recursive, meaning these attributes contain their own DRUIDs. Henge can recurse through these, allowing you to mint unique identifiers for arbitrary nested data structures.
+
+A DRUID is ultimately the result of a digest operation (such as `md5` or `sha256`) on some data. Because DRUIDs are computed deterministically from the item, they represent globally unique identifiers. If you insert the same item repeatedly, it will produce the same DRUID -- this is true across henges as long as they share a data schema.
+
+## Persisting Data
+
+### In-memory (default)
+
+Use a Python `dict` as the database for testing or ephemeral use:
+
+```python
+h = henge.Henge(database={}, schemas=schemas)
+```
+
+### SQLite backend
+
+For persistent storage with SQLite:
+
+```python
+from sqlitedict import SqliteDict
+
+mydict = SqliteDict('./my_db.sqlite', autocommit=True)
+h = henge.Henge(mydict, schemas=schemas)
+```
+
+Requires: `pip install sqlitedict`
+
+### MongoDB backend
+
+For production use with MongoDB:
+
+1. **Start MongoDB with Docker:**
+
+```bash
+docker run --network="host" mongo
+```
+
+For persistent storage, mount a volume to `/data/db`:
+
+```bash
+docker run -it --network="host" -v /path/to/data:/data/db mongo
+```
+
+2. **Connect henge to MongoDB:**
+
+```python
+import henge
+
+h = henge.Henge(henge.connect_mongo(), schemas=schemas)
+```
+
+Requires: `pip install pymongo mongodict`

henge-0.2.3.dist-info/RECORD

@@ -0,0 +1,11 @@
+henge/__init__.py,sha256=PBJsthV7TCS3Wu-4fLxBIaBFf_GwcnN3HyWELHuWdAI,246
+henge/_version.py,sha256=PNiDER4qM19h9zdsdfgKt2_dT4WgYK7EguJ8RU2qA_g,22
+henge/const.py,sha256=0t3EgqdjmKBd-zu5L8AJnGoWv0T3sAtvXf-3b62Dd-Y,194
+henge/deprecated.py,sha256=C8eINR2gWCiNaN2b3gbFYn8jfJ0ftJm8a_fIgVVVzXc,11248
+henge/henge.py,sha256=UHQCXNcGs0zsOXarll3ZCzddb6pWqGWml5ZqBXSM6WU,24088
+henge/scconf.py,sha256=b201HzE_l3hO0JgusQnI2XhdLayWgXEoqeaZzJwSke4,12132
+henge-0.2.3.dist-info/licenses/LICENSE.txt,sha256=oB6ZGDa4kcznznJKJsLLFFcOZyi8Y6e2Jv0rJozgp-I,1269
+henge-0.2.3.dist-info/METADATA,sha256=Qhf7hrZyhXJRpJVnDdi2bn717BNsScFKk1UncE8JGHo,4402
+henge-0.2.3.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+henge-0.2.3.dist-info/top_level.txt,sha256=QyovlLuKhhKP1r8bMVmxLdke9F6PZFIN7VlkzvB0xIQ,6
+henge-0.2.3.dist-info/RECORD,,

{henge-0.2.2.dist-info → henge-0.2.3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.42.0)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

henge-0.2.2.dist-info/METADATA

@@ -1,28 +0,0 @@
-Metadata-Version: 2.1
-Name: henge
-Version: 0.2.2
-Summary: Storage and retrieval of object-derived, decomposable recursive unique identifiers.
-Home-page: https://databio.org
-Author: Nathan Sheffield
-Author-email: nathan@code.databio.org
-License: BSD2
-Classifier: Development Status :: 4 - Beta
-Classifier: License :: OSI Approved :: BSD License
-Classifier: Programming Language :: Python :: 3.7
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Topic :: System :: Distributed Computing
-Description-Content-Type: text/markdown
-License-File: LICENSE.txt
-Requires-Dist: jsonschema
-Requires-Dist: ubiquerg >=0.5.2
-Requires-Dist: yacman >=0.6.7
-
-[![Build Status](https://travis-ci.com/databio/henge.svg?branch=master)](https://travis-ci.com/databio/henge)
-
-# Henge 
-
-Henge is a Python package that builds backends for generic decomposable recursive unique identifiers (or, *DRUIDs*). It is intended to be used as a building block for sequence collections (see the [seqcol package](https://github.com/databio/seqcol)), and also for other data types that need content-derived identifiers.
-
-Documentation at [http://henge.databio.org](http://henge.databio.org).

henge-0.2.2.dist-info/RECORD

@@ -1,11 +0,0 @@
-henge/__init__.py,sha256=_S3iwLd7FBA7rPmmhPGyChxkSBKdOEiR9jV3eyfD79Q,245
-henge/_version.py,sha256=HfjVOrpTnmZ-xVFCYSVmX50EXaBQeJteUHG-PD6iQs8,22
-henge/const.py,sha256=0t3EgqdjmKBd-zu5L8AJnGoWv0T3sAtvXf-3b62Dd-Y,194
-henge/deprecated.py,sha256=C8eINR2gWCiNaN2b3gbFYn8jfJ0ftJm8a_fIgVVVzXc,11248
-henge/henge.py,sha256=RDneIEpR7m_ygq1cERZsXJHW9bsX0jXZ7G5PxCTkfpU,24007
-henge-0.2.2.dist-info/LICENSE.txt,sha256=oB6ZGDa4kcznznJKJsLLFFcOZyi8Y6e2Jv0rJozgp-I,1269
-henge-0.2.2.dist-info/METADATA,sha256=5I9rS8s2if6gD2s0nDqNjORaFGwb9GOVeIp_W0yVIJ8,1266
-henge-0.2.2.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-henge-0.2.2.dist-info/entry_points.txt,sha256=c2OKgrH1a5Cx2osbUFSe9NFK8CbN82lPPsi4wry77_M,61
-henge-0.2.2.dist-info/top_level.txt,sha256=QyovlLuKhhKP1r8bMVmxLdke9F6PZFIN7VlkzvB0xIQ,6
-henge-0.2.2.dist-info/RECORD,,

henge-0.2.2.dist-info/entry_points.txt

@@ -1,2 +0,0 @@
-[console_scripts]
-packagename = packagename.packagename:main