hadro 0.0.4__tar.gz

hadro-0.0.4/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Justin Joyce
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

hadro-0.0.4/NOTICES

@@ -0,0 +1,5 @@
+HadroDB is based on [CaskDB](https://github.com/avinassh/py-caskdb) which is based on [Bitcask](https://riak.com/assets/bitcask-intro.pdf).
+
+[ormsgpack](https://github.com/aviramha/ormsgpack)
+
+[CityHash](https://github.com/escherba/python-cityhash)

hadro-0.0.4/PKG-INFO

@@ -0,0 +1,42 @@
+Metadata-Version: 2.1
+Name: hadro
+Version: 0.0.4
+Summary: Storage Engine
+Home-page: https://github.com/mabel-dev/hadrodb/
+Author: @joocer
+Author-email: justin.joyce@joocer.com
+Maintainer: @joocer
+Description-Content-Type: text/markdown
+License-File: LICENSE
+License-File: NOTICES
+
+<div align="center">
+
+![HadroDB](https://raw.githubusercontent.com/mabel-dev/hadrodb/main/hadro.png)
+
+HadroDB is a database storage engine for [Opteryx](https://opteryx.dev).
+
+[![PyPI Latest Release](https://img.shields.io/pypi/v/hadrodb.svg)](https://pypi.org/project/hadrodb/)
+[![codecov](https://codecov.io/gh/mabel-dev/hadrodb/branch/main/graph/badge.svg?token=nl9JwOVdPs)](https://codecov.io/gh/mabel-dev/hadrodb)
+[![Downloads](https://static.pepy.tech/badge/hadrodb)](https://pepy.tech/project/hadrodb)
+
+</div>
+
+## License
+
+[![License](https://img.shields.io/badge/License-MIT-blue.svg)](https://github.com/mabel-dev/hadrodb/blob/main/LICENSE)
+[![Notices](https://img.shields.io/badge/-Notices-darkgreen.svg)](https://github.com/mabel-dev/hadrodb/blob/main/NOTICES)
+
+## Status
+
+[![Status](https://img.shields.io/badge/Status-alpha-orange)](https://github.com/mabel-dev/opteryx)
+
+HadroDB is in alpha. Alpha means different things to different people, to us, being alpha means:
+
+- Interfaces may be significantly changed
+- The data file format is unstable
+- Expected functionality is missing
+- Things that worked yesterday, don't work today
+- The results of the system may be unreliable
+
+As such, we really don't recommend using HadroDB anywhere where your data matters.

hadro-0.0.4/README.md

@@ -0,0 +1,30 @@
+<div align="center">
+
+![HadroDB](https://raw.githubusercontent.com/mabel-dev/hadrodb/main/hadro.png)
+
+HadroDB is a database storage engine for [Opteryx](https://opteryx.dev).
+
+[![PyPI Latest Release](https://img.shields.io/pypi/v/hadrodb.svg)](https://pypi.org/project/hadrodb/)
+[![codecov](https://codecov.io/gh/mabel-dev/hadrodb/branch/main/graph/badge.svg?token=nl9JwOVdPs)](https://codecov.io/gh/mabel-dev/hadrodb)
+[![Downloads](https://static.pepy.tech/badge/hadrodb)](https://pepy.tech/project/hadrodb)
+
+</div>
+
+## License
+
+[![License](https://img.shields.io/badge/License-MIT-blue.svg)](https://github.com/mabel-dev/hadrodb/blob/main/LICENSE)
+[![Notices](https://img.shields.io/badge/-Notices-darkgreen.svg)](https://github.com/mabel-dev/hadrodb/blob/main/NOTICES)
+
+## Status
+
+[![Status](https://img.shields.io/badge/Status-alpha-orange)](https://github.com/mabel-dev/opteryx)
+
+HadroDB is in alpha. Alpha means different things to different people, to us, being alpha means:
+
+- Interfaces may be significantly changed
+- The data file format is unstable
+- Expected functionality is missing
+- Things that worked yesterday, don't work today
+- The results of the system may be unreliable
+
+As such, we really don't recommend using HadroDB anywhere where your data matters.

hadro-0.0.4/hadro/__init__.py

@@ -0,0 +1,3 @@
+from hadro.engine import HadroDB
+from hadro.version import __author__
+from hadro.version import __version__

hadro-0.0.4/hadro/config.py

@@ -0,0 +1,24 @@
+from enum import Enum
+from enum import auto
+
+
+class ConsistencyMode(int, Enum):
+    """
+    Aggressive will ensure each write is flushed before moving to the next, Relaxed
+    allows the OS to handle write flushes.
+
+    Aggressive has significant impact on write performance, but significantly reduces
+    the chance of data loss due to system crashes.
+
+    Relaxed is faster (10x), but if the system crashes, your data may be in an
+    inconsistent state.
+
+    Aggressive is the default (safe default), but there are some scenarios where a
+    relaxed approach to write consistency is okay or acceptable.
+    """
+
+    AGGRESSIVE = auto()
+    RELAXED = auto()
+
+
+WRITE_CONSISTENCY: ConsistencyMode = ConsistencyMode.RELAXED

hadro-0.0.4/hadro/engine.py

@@ -0,0 +1,197 @@
+"""
+
+Typical usage example:
+
+    disk: DiskStorage = DiskStore(file_name="books.db")
+    disk.set(key="othello", value="shakespeare")
+    author: str = disk.get("othello")
+    # it also supports dictionary style API too:
+    disk["hamlet"] = "shakespeare"
+"""
+import io
+import os.path
+import struct
+import typing
+from collections import namedtuple
+
+from hadro.config import WRITE_CONSISTENCY
+from hadro.config import ConsistencyMode
+from orso import logging
+from orso.row import Row
+
+logging.set_log_name("MESOS")
+logger = logging.get_logger()
+logger.setLevel(5)
+
+DELETED_FLAG: int = 1
+
+RecordHeader = namedtuple("RecordHeader", ["flags", "size"])
+
+
+# DiskStorage is a Log-Structured Hash Table as described in the BitCask paper. We
+# keep appending the data to a file, like a log. DiskStorage maintains an in-memory
+# hash table called KeyDir, which keeps the row's location on the disk.
+#
+# The idea is simple yet brilliant:
+#   - Write the record to the disk
+#   - Update the internal hash table to point to that byte offset
+#   - Whenever we get a read request, check the internal hash table for the address,
+#       fetch that and return
+#
+# KeyDir does not store values, only their locations.
+#
+# The above approach solves a lot of problems:
+#   - Writes are insanely fast since you are just appending to the file
+#   - Reads are insanely fast since you do only one disk seek. In B-Tree backed
+#       storage, there could be 2-3 disk seeks
+#
+# However, there are drawbacks too:
+#   - We need to maintain an in-memory hash table KeyDir. A database with a large
+#       number of keys would require more RAM
+#   - Since we need to build the KeyDir at initialisation, it will affect the startup
+#       time too
+#   - Deleted keys need to be purged from the file to reduce the file size
+#
+# Read the paper for more details: https://riak.com/assets/bitcask-intro.pdf
+
+
+class HadroDB:
+    """
+    Implements the KV store on the disk
+
+    Args:
+        file_name (str): name of the file where all the data will be written. Just
+            passing the file name will save the data in the current directory. You may
+            pass the full file location too.
+
+    Attributes:
+        file_name (str): name of the file where all the data will be written. Just
+            passing the file name will save the data in the current directory. You may
+            pass the full file location too.
+        file (typing.BinaryIO): file object pointing the file_name
+        write_position (int): current cursor position in the file where the data can be
+            written
+        key_dir (dict[str, KeyEntry]): is a map of key and KeyEntry being the value.
+            KeyEntry contains the position of the byte offset in the file where the
+            value exists. key_dir map acts as in-memory index to fetch the values
+            quickly from the disk
+    """
+
+    def __init__(self, collection: typing.Union[str, None] = None):
+        logger.warning("HadroDB is experimental and not recommended for use.")
+        self.collection: str = collection
+        self.file_name: str = collection + "/00000000.data"
+        self._schema_file: str = collection + "/00000000.schema"
+        self.write_position: int = 0
+        self.key_dir: dict[bytes, Row] = {}
+
+        if collection is None:
+            raise ValueError("HadroDB requires a collection name")
+        # if the collection exists, it must be a folder, not a file
+        if os.path.exists(collection):
+            if not os.path.isdir(collection):
+                raise ValueError("Collection must be a folder")
+            # if the file exists already, then we will load the key_dir
+        #            self._init_key_dir()
+        else:
+            os.makedirs(collection, exist_ok=True)
+
+        #        if os.path.exists(self._schema_file):
+        #            load the schema
+
+        # we open the file in `a+b` mode:
+        # a - says the writes are append only. `a+` means we want append and read
+        # b - says that we are operating the file in binary mode (as opposed to the
+        #     default string mode)
+        self.file: typing.BinaryIO = open(self.file_name, "a+b")
+        self.fileno = self.file.fileno()
+
+        schema = {
+            "id": {"type": "SMALLINT", "nullable": False},
+            "planetId": {"type": "SMALLINT", "nullable": False},
+            "name": {"type": "VARCHAR", "nullable": False},
+            "gm": {"type": "FLOAT", "nullable": False},
+            "radius": {"type": "FLOAT", "nullable": False},
+            "density": {"type": "FLOAT", "nullable": True},
+            "magnitude": {"type": "FLOAT", "nullable": True},
+            "albedo": {"type": "FLOAT", "nullable": True},
+        }
+
+        self.rows = Row.create_class(schema)
+
+    def append(self, record) -> None:
+        if isinstance(record, dict):
+            _record = record.values()
+        else:
+            _record = record
+
+        record = self.rows(_record)
+        # test it matches the schema
+
+        bytes_to_write = record.to_bytes()
+        self._write(bytes_to_write)
+
+        # update indices index
+        #
+
+        self.write_position += len(bytes_to_write)
+
+    def scan(self, columns=None, predicates=None):
+        block_size: int = 8 * 1024 * 1024  # read 1Mb at a time
+        self.file.seek(0, 0)
+
+        # TODO: read file header
+
+        buffer = io.BufferedReader(self.file, block_size)  # type: ignore
+
+        header_bytes = buffer.read(5)
+        flags, size = struct.unpack(">BI", header_bytes)
+        block_start = 5  # start of the current block
+
+        while size > 0:
+            if block_start + size > block_size:
+                # The current record spans multiple blocks, so read the rest of it in the next block
+                remaining_size = size - (block_size - block_start)
+                data_bytes = bytearray(
+                    buffer.read(block_size - block_start)
+                )  # read the remaining bytes in the current block
+                while remaining_size > 0:
+                    # Read the remaining bytes in subsequent blocks
+                    block_bytes = buffer.read(min(remaining_size, block_size))
+                    data_bytes += block_bytes
+                    remaining_size -= len(block_bytes)
+                    block_start = len(block_bytes)
+            else:
+                # The current record fits in the current block, so just read it
+                data_bytes = bytearray(buffer.read(size))
+                block_start += size
+
+            if flags & DELETED_FLAG == 0:
+                yield self.rows.from_bytes(data_bytes)
+
+            # Read the size of the next record
+            header_bytes = buffer.read(5)
+            if len(header_bytes) == 0:
+                break
+            flags, size = struct.unpack(">BI", header_bytes)
+            block_start += 5  # add the size of the size field to the start of the next block
+
+    def _write(self, data: bytes) -> None:
+        # saving stuff to a file reliably is hard!
+        # if you would like to explore and learn more, then
+        # start from here: https://danluu.com/file-consistency/
+        # and read this too: https://lwn.net/Articles/457667/
+        os.write(self.fileno, data)
+
+        if WRITE_CONSISTENCY == ConsistencyMode.AGGRESSIVE:
+            # calling fsync after every write is important, this assures that our writes
+            # are actually persisted to the disk
+            os.fsync(self.fileno)
+
+    def close(self) -> None:
+        # before we close the file, we need to safely write the contents in the buffers
+        # to the disk. Check documentation of DiskStorage._write() to understand
+        # following the operations
+        self.file.flush()
+        os.fsync(self.fileno)
+        self.file.close()

hadro-0.0.4/hadro/schema.py

@@ -0,0 +1,79 @@
+class Field:
+    def __init__(self, name, ftype, nullable=True):
+        self.name = name
+        self.ftype = ftype
+        self.nullable = nullable
+
+
+class Index:
+    def __init__(self, name, fields):
+        self.name = name
+        self.fields = fields
+
+
+class TableSchema:
+    def __init__(self, fields=None, primary_key=None, indexes=None):
+        self._fields = {}
+        self.primary_key = primary_key
+        self.indexes = indexes or []
+
+        if fields is not None:
+            for name, field in fields.items():
+                self.add_field(name, field["type"], field.get("nullable", True))
+
+    def add_field(self, name_or_field, field_type=None, nullable=True):
+        if isinstance(name_or_field, Field):
+            name = name_or_field.name
+            field = name_or_field
+        else:
+            name = name_or_field
+            field = Field(name, field_type, nullable)
+        self._fields[name] = field
+
+    def remove_field(self, name):
+        if name in self._fields:
+            del self._fields[name]
+
+    def field(self, i):
+        if isinstance(i, int):
+            return self._fields[list(self._fields.keys())[i]]
+        else:
+            return self._fields[i]
+
+    @property
+    def num_fields(self):
+        return len(self._fields)
+
+    @property
+    def names(self):
+        return list(self._fields.keys())
+
+    @property
+    def types(self):
+        return [field["type"] for field in self._fields.values()]
+
+    def to_dict(self):
+        return {
+            "fields": {
+                name: {"type": field["type"], "nullable": field["nullable"]}
+                for name, field in self._fields.items()
+            },
+            "primaryKey": self.primary_key,
+            "indexes": [index.to_dict() for index in self.indexes],
+        }
+
+    def validate(self, data):
+        if not isinstance(data, dict):
+            return False
+
+        # Check that all required fields are present
+        for field_name, field in self._fields.items():
+            if not field["nullable"] and field_name not in data:
+                return False
+
+        # Check that all fields have the correct type
+        for field_name, field in self._fields.items():
+            if field_name in data and not isinstance(data[field_name], field["type"]):
+                return False
+
+        return True

hadro-0.0.4/hadro/version.py

@@ -0,0 +1,2 @@
+__version__ = "0.0.4"
+__author__ = "@joocer"

hadro-0.0.4/hadro.egg-info/PKG-INFO

@@ -0,0 +1,42 @@
+Metadata-Version: 2.1
+Name: hadro
+Version: 0.0.4
+Summary: Storage Engine
+Home-page: https://github.com/mabel-dev/hadrodb/
+Author: @joocer
+Author-email: justin.joyce@joocer.com
+Maintainer: @joocer
+Description-Content-Type: text/markdown
+License-File: LICENSE
+License-File: NOTICES
+
+<div align="center">
+
+![HadroDB](https://raw.githubusercontent.com/mabel-dev/hadrodb/main/hadro.png)
+
+HadroDB is a database storage engine for [Opteryx](https://opteryx.dev).
+
+[![PyPI Latest Release](https://img.shields.io/pypi/v/hadrodb.svg)](https://pypi.org/project/hadrodb/)
+[![codecov](https://codecov.io/gh/mabel-dev/hadrodb/branch/main/graph/badge.svg?token=nl9JwOVdPs)](https://codecov.io/gh/mabel-dev/hadrodb)
+[![Downloads](https://static.pepy.tech/badge/hadrodb)](https://pepy.tech/project/hadrodb)
+
+</div>
+
+## License
+
+[![License](https://img.shields.io/badge/License-MIT-blue.svg)](https://github.com/mabel-dev/hadrodb/blob/main/LICENSE)
+[![Notices](https://img.shields.io/badge/-Notices-darkgreen.svg)](https://github.com/mabel-dev/hadrodb/blob/main/NOTICES)
+
+## Status
+
+[![Status](https://img.shields.io/badge/Status-alpha-orange)](https://github.com/mabel-dev/opteryx)
+
+HadroDB is in alpha. Alpha means different things to different people, to us, being alpha means:
+
+- Interfaces may be significantly changed
+- The data file format is unstable
+- Expected functionality is missing
+- Things that worked yesterday, don't work today
+- The results of the system may be unreliable
+
+As such, we really don't recommend using HadroDB anywhere where your data matters.

hadro-0.0.4/hadro.egg-info/SOURCES.txt

@@ -0,0 +1,16 @@
+LICENSE
+NOTICES
+README.md
+pyproject.toml
+setup.py
+hadro/__init__.py
+hadro/config.py
+hadro/engine.py
+hadro/schema.py
+hadro/version.py
+hadro.egg-info/PKG-INFO
+hadro.egg-info/SOURCES.txt
+hadro.egg-info/dependency_links.txt
+hadro.egg-info/requires.txt
+hadro.egg-info/top_level.txt
+tests/test_interface.py

hadro-0.0.4/hadro.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

hadro-0.0.4/hadro.egg-info/requires.txt

@@ -0,0 +1,2 @@
+ormsgpack
+orso

hadro-0.0.4/hadro.egg-info/top_level.txt

@@ -0,0 +1 @@
+hadro

hadro-0.0.4/pyproject.toml

@@ -0,0 +1,11 @@
+[tool.black]
+line-length = 100
+target-version = ['py310']
+
+[tool.isort]
+profile = "black"
+src_paths = ["hadrodb"]
+line_length = 100
+multi_line_output = 9
+force_single_line = true
+float_to_top = true

hadro-0.0.4/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

hadro-0.0.4/setup.py

@@ -0,0 +1,37 @@
+from setuptools import find_packages
+from setuptools import setup
+
+LIBRARY = "hadro"
+
+
+__version__ = "notset"
+with open(f"{LIBRARY}/version.py", mode="r") as v:
+    vers = v.read()
+exec(vers)  # nosec
+
+with open("README.md", mode="r", encoding="UTF8") as rm:
+    long_description = rm.read()
+
+try:
+    with open("requirements.txt", "r") as f:
+        required = f.read().splitlines()
+except:
+    with open(f"{LIBRARY}.egg-info/requires.txt", "r") as f:
+        required = f.read().splitlines()
+
+
+setup_config = {
+    "name": LIBRARY,
+    "version": __version__,
+    "description": "Storage Engine",
+    "long_description": long_description,
+    "long_description_content_type": "text/markdown",
+    "maintainer": "@joocer",
+    "author": "@joocer",
+    "author_email": "justin.joyce@joocer.com",
+    "packages": find_packages(include=[LIBRARY, f"{LIBRARY}.*"]),
+    "url": "https://github.com/mabel-dev/hadrodb/",
+    "install_requires": required,
+}
+
+setup(**setup_config)

hadro-0.0.4/tests/test_interface.py

@@ -0,0 +1,90 @@
+import os
+import shutil
+import sys
+import time
+
+sys.path.insert(1, os.path.join(sys.path[0], ".."))
+sys.path.insert(1, os.path.join(sys.path[0], "../.."))
+
+from hadro import HadroDB  # isort: skip
+
+
+TEST_COLLECTION = "test_123"
+
+
+def create_doc(docs):
+    docid = str(time.monotonic_ns())
+    doc = {"document": docid}
+    docs[docid] = doc
+    return docid, doc
+
+
+def test_interface():
+    # initialize the dataset
+    if os.path.exists(TEST_COLLECTION):  # pragma: no cover
+        shutil.rmtree(TEST_COLLECTION, ignore_errors=True)
+
+    # create a copy of the docs for us to compare against
+    comparision_set_of_docs = {}
+
+    # OPEN COLLECTION
+    hadro = HadroDB(TEST_COLLECTION)
+
+    # SET DOCUMENT IN COLLECTION
+    # subscript syntax
+    docid, doc = create_doc(comparision_set_of_docs)
+    hadro.append(doc)
+
+    docid, doc = create_doc(comparision_set_of_docs)
+    # set syntax
+    hadro.append(doc)
+
+    # add syntax
+    docid = hadro.append(doc)
+    comparision_set_of_docs[docid] = doc
+
+    # GET DOCUMENT FROM COLLECTION
+    # subscript, single doc syntax
+    #    doc = hadro[docid]
+    #    assert doc == comparision_set_of_docs[docid]
+
+    # subscript, list syntax
+    #    list_of_docs = hadro[comparision_set_of_docs.keys()]
+    #    assert len(list_of_docs) == len(comparision_set_of_docs)
+
+    # get syntax
+    #    doc = hadro.get(docid)
+    #    assert doc == comparision_set_of_docs[docid]
+
+    # GET COLLECTION SIZE
+    #    assert len(hadro) == len(comparision_set_of_docs)
+
+    # GET KEYS IN COLLECTION
+    #    keys = hadro.keys()
+    #    assert len(keys) == len(comparision_set_of_docs)
+    #    for key in keys:
+    #        assert hadro[key] == comparision_set_of_docs[key]
+
+    """
+# DELETE DOCUMENT FROM COLLECTION
+del hadro[id]
+hadro.del(id[, id...])
+
+# CONTAINMENT
+id in hadro
+hadro.contains(id[, id ...])
+
+# FILTER COLLECTION
+filtered_collection = hadro.where(predicate)
+
+
+"""
+
+    hadro.close()
+
+    # tidy up
+    if os.path.exists(TEST_COLLECTION):  # pragma: no cover
+        shutil.rmtree(TEST_COLLECTION, ignore_errors=True)
+
+
+test_interface()