ob-dump-reader 0.0.0.dev0__py3-none-any.whl

ob_dump_reader/__init__.py

@@ -0,0 +1,93 @@
+from collections.abc import Buffer
+from os import PathLike
+from typing import Callable
+from dataclasses import dataclass
+
+import lmdb
+
+from ob_dump_reader._constants import (
+    KEY_TYPE_DATA,
+    KEY_TYPE_RELATION,
+    RELATION_DIRECTION_FORWARD,
+)
+from ob_dump_reader.decode_helpers import read_uint32be
+
+__all__ = (
+    "ObRecord",
+    "read_objectbox_records",
+    "read_objectbox_to_many_targets",
+)
+
+
+@dataclass
+class ObRecord:
+    entity_id: int
+    object_id: int
+    data: Buffer
+
+
+def read_objectbox_records(
+    objectbox_dir: PathLike,
+    on_record: Callable[[ObRecord], None],
+) -> None:
+    # readonly=True is enough on its own — LMDB's MVCC design lets any number
+    # of readers run safely alongside one concurrent writer, in any process,
+    # so no write transaction is needed just to open the store.
+    with lmdb.Environment(
+        str(objectbox_dir),
+        map_size=512 * 1024 * 1024,
+        max_dbs=4,
+        readonly=True,
+    ) as db:
+        with db.begin(write=False) as txn:
+            with txn.cursor() as cursor:
+                for key, value in cursor:
+                    if key and len(key) == 8 and key[0] == KEY_TYPE_DATA:
+                        entity_id = key[3] // 4
+                        object_id = read_uint32be(key, 4)
+
+                        on_record(
+                            ObRecord(
+                                entity_id=entity_id,
+                                object_id=object_id,
+                                data=bytes(value),  # copy out of the mmap'd page
+                            )
+                        )
+
+
+def read_objectbox_to_many_targets(
+    objectbox_dir: PathLike, relation_id: int, source_object_id: int
+) -> list[int]:
+    """Resolves a `ToMany` relation's forward-direction target ids.
+
+    Not part of the FlatBuffers table at all — a separate LMDB key range,
+    12 bytes: type(0x08) + 2 reserved bytes + ((relation_id << 2) |
+    direction) + source_id (u32 BE) + target_id (u32 BE), value always
+    empty. `direction` 0 is the declared forward direction; 2 is an
+    auto-maintained reverse index for ObjectBox's own query engine, not
+    needed for a one-directional dump.
+    """
+    prefix = bytes(
+        [
+            KEY_TYPE_RELATION,
+            0x00,
+            0x00,
+            (relation_id << 2) | RELATION_DIRECTION_FORWARD,
+        ]
+    ) + source_object_id.to_bytes(4, "big")
+
+    targets: list[int] = []
+    with lmdb.Environment(
+        str(objectbox_dir),
+        map_size=512 * 1024 * 1024,
+        max_dbs=4,
+        readonly=True,
+    ) as db:
+        with db.begin(write=False) as txn:
+            with txn.cursor() as cursor:
+                if cursor.set_range(prefix):
+                    for key, _ in cursor:
+                        if bytes(key[:8]) != prefix:
+                            break
+                        targets.append(read_uint32be(key, 8))
+    return targets

ob_dump_reader/__main__.py

@@ -0,0 +1,30 @@
+import argparse
+
+import ob_dump_reader as ob
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        prog="ob_dump_reader",
+        description=(
+            "Walk an ObjectBox LMDB store and print each record's entity id, "
+            "object id, and raw FlatBuffers data length. Decoding the data "
+            "itself needs a flatc-generated schema — see the package README."
+        ),
+    )
+    parser.add_argument(
+        "objectbox_dir",
+        help="path to the ObjectBox data directory (containing data.mdb)",
+    )
+    args = parser.parse_args()
+
+    def on_record(record: ob.ObRecord) -> None:
+        print(
+            f"entity={record.entity_id} object={record.object_id} bytes={len(record.data)}"
+        )
+
+    ob.read_objectbox_records(args.objectbox_dir, on_record)
+
+
+if __name__ == "__main__":
+    main()

ob_dump_reader/_constants.py

@@ -0,0 +1,4 @@
+KEY_TYPE_DATA: int = 0x18
+KEY_TYPE_RELATION: int = 0x08
+RELATION_DIRECTION_FORWARD: int = 0
+RELATION_DIRECTION_BACKWARD: int = 2

ob_dump_reader/aio.py

@@ -0,0 +1,99 @@
+from collections.abc import Buffer
+from os import PathLike
+from typing import Any, Callable, Coroutine
+from dataclasses import dataclass
+
+import lmdb
+import lmdb.aio
+
+from ob_dump_reader._constants import (
+    KEY_TYPE_DATA,
+    KEY_TYPE_RELATION,
+    RELATION_DIRECTION_FORWARD,
+)
+from ob_dump_reader.decode_helpers import read_uint32be
+
+__all__ = (
+    "ObRecord",
+    "read_objectbox_records",
+    "read_objectbox_to_many_targets",
+)
+
+
+@dataclass
+class ObRecord:
+    entity_id: int
+    object_id: int
+    data: Buffer
+
+
+async def read_objectbox_records(
+    objectbox_dir: PathLike | str,
+    on_record: Callable[[ObRecord], Coroutine[Any, Any, None]],
+) -> None:
+    # AsyncEnvironment wraps a plain (sync) Environment — it doesn't take a
+    # path/map_size/etc. itself (lmdb.aio's own docstring example). Cursor
+    # has no __aiter__, and its iternext() collects every remaining entry
+    # into a list before returning (nothing streams past the executor
+    # boundary) — stepping via first()/next() instead keeps this a true
+    # per-record callback, not "load the whole database, then callback".
+    env = lmdb.Environment(
+        str(objectbox_dir),
+        map_size=512 * 1024 * 1024,
+        max_dbs=4,
+        readonly=True,
+    )
+    aenv = lmdb.aio.wrap(env)
+    async with aenv:
+        async with aenv.begin(write=False) as txn:
+            async with txn.cursor() as cursor:
+                has_item = await cursor.first()
+                while has_item:
+                    key, value = cursor.item()
+                    if key and len(key) == 8 and key[0] == KEY_TYPE_DATA:
+                        entity_id = key[3] // 4
+                        object_id = read_uint32be(key, 4)
+
+                        await on_record(
+                            ObRecord(
+                                entity_id=entity_id,
+                                object_id=object_id,
+                                data=bytes(value),
+                            )
+                        )
+                    has_item = await cursor.next()
+
+
+async def read_objectbox_to_many_targets(
+    objectbox_dir: PathLike | str, relation_id: int, source_object_id: int
+) -> list[int]:
+    """Async counterpart of `ob_dump_reader.read_objectbox_to_many_targets` —
+    see its docstring for the key format."""
+    prefix = bytes(
+        [
+            KEY_TYPE_RELATION,
+            0x00,
+            0x00,
+            (relation_id << 2) | RELATION_DIRECTION_FORWARD,
+        ]
+    ) + source_object_id.to_bytes(4, "big")
+
+    targets: list[int] = []
+    env = lmdb.Environment(
+        str(objectbox_dir),
+        map_size=512 * 1024 * 1024,
+        max_dbs=4,
+        readonly=True,
+    )
+    aenv = lmdb.aio.wrap(env)
+    async with aenv:
+        async with aenv.begin(write=False) as txn:
+            async with txn.cursor() as cursor:
+                has_item = await cursor.set_range(prefix)
+                while has_item:
+                    key, _ = cursor.item()
+                    if bytes(key[:8]) != prefix:
+                        break
+                    targets.append(read_uint32be(key, 8))
+                    has_item = await cursor.next()
+    return targets

ob_dump_reader/decode_helpers.py

@@ -0,0 +1,98 @@
+from collections.abc import Buffer
+
+try:
+    import orjson as json
+except ImportError:
+    import json
+from typing import Any
+from flatbuffers import flexbuffers
+
+__all__ = (
+    "uint32be",
+    "read_uint32be",
+    "bytes_to_hex",
+    "bytes_to_uuid_string",
+    "try_parse_json_string",
+    "decode_flex",
+)
+
+
+def uint32be(v: int) -> list[int]:
+    return [
+        (v >> 24) & 0xFF,
+        (v >> 16) & 0xFF,
+        (v >> 8) & 0xFF,
+        v & 0xFF,
+    ]
+
+
+def read_uint32be(buf: Buffer, offset: int) -> int:
+    return (
+        (buf[offset] << 24)
+        | (buf[offset + 1] << 16)
+        | (buf[offset + 2] << 8)
+        | buf[offset + 3]
+    )
+
+
+def bytes_to_hex(b: Buffer | list[int]) -> str:
+    return bytes(b).hex()
+
+
+def bytes_to_uuid_string(b: bytes | list[int]) -> str:
+    hex_str = bytes(b).hex()
+    if len(hex_str) != 32:
+        return hex_str
+    return (
+        f"{hex_str[0:8]}-{hex_str[8:12]}-"
+        f"{hex_str[12:16]}-{hex_str[16:20]}-"
+        f"{hex_str[20:32]}"
+    )
+
+
+def try_parse_json_string(s: str) -> Any:
+    try:
+        return json.loads(s)
+    except json.JSONDecodeError:
+        return s
+
+
+def _flex_ref_to_native(ref: Any) -> Any:
+    # flatbuffers.flexbuffers.Ref has no `.json`/`str()` serialization of its
+    # own (an earlier version of this function assumed `str(root)` gave JSON
+    # — it gives a debug repr like "Ref(buf[21:], ...)", not decodable data;
+    # caught by actually calling this against real FlexBuffers bytes, not
+    # assumed). Is*/As* are properties, not methods.
+    if ref.IsNull:
+        return None
+    if ref.IsBool:
+        return ref.AsBool
+    if ref.IsInt:
+        return ref.AsInt
+    if ref.IsFloat:
+        return ref.AsFloat
+    if ref.IsString:
+        return ref.AsString
+    if ref.IsBlob:
+        return bytes_to_hex(ref.AsBlob)
+    if ref.IsMap:
+        m = ref.AsMap
+        return {
+            key.AsKey: _flex_ref_to_native(value)
+            for key, value in zip(m.Keys, m.Values)
+        }
+    if ref.IsVector or ref.IsTypedVector or ref.IsFixedTypedVector:
+        vec = (
+            ref.AsVector
+            if ref.IsVector
+            else ref.AsTypedVector
+            if ref.IsTypedVector
+            else ref.AsFixedTypedVector
+        )
+        return [_flex_ref_to_native(vec[i]) for i in range(len(vec))]
+    raise ValueError(f"Unsupported FlexBuffers value: {ref}")
+
+
+def decode_flex(b: Buffer) -> Any:
+    root = flexbuffers.GetRoot(bytes(b))
+    return _flex_ref_to_native(root)

ob_dump_reader-0.0.0.dev0.dist-info/METADATA

@@ -0,0 +1,174 @@
+Metadata-Version: 2.4
+Name: ob-dump-reader
+Version: 0.0.0.dev0
+Summary: Minimal ObjectBox LMDB reader toolkit
+Author: o-murphy
+Project-URL: Homepage, https://github.com/o-murphy/ob-dump
+Project-URL: Bug Reports, https://github.com/o-murphy/ob-dump/issues
+Project-URL: Source, https://github.com/o-murphy/ob-dump
+Project-URL: Changelog, https://github.com/o-murphy/ob-dump/blob/main/CHANGELOG.md
+Keywords: objectbox,lmdb,flatbuffers
+Classifier: Intended Audience :: Developers
+Classifier: Natural Language :: English
+Classifier: Programming Language :: Python
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: flatbuffers>=25.12.19
+Requires-Dist: lmdb>=2.2.1
+Provides-Extra: orjson
+Requires-Dist: orjson==3.11.9; extra == "orjson"
+Dynamic: license-file
+
+# ob-dump-reader
+
+![GitHub License](https://img.shields.io/github/license/o-murphy/ob-dump)
+![PyPI Version](https://img.shields.io/pypi/v/ob_dump_reader?logo=pypi)
+
+Minimal ObjectBox LMDB reader toolkit for Python. Its core job: walk a
+`data.mdb` file and hand you each stored object's raw FlatBuffers table
+bytes, plus its entity id and object id — no per-field FlatBuffers or
+schema knowledge for that part, just [`lmdb`](https://pypi.org/project/lmdb/)
+(the [py-lmdb](https://github.com/jnwatson/py-lmdb) bindings). It also
+covers the handful of things `flatc --python` output can't: `ToMany`
+relations (a separate LMDB structure, not a table field) and `Flex`/
+`ExternalPropertyType` fields (`flatc` only knows the base FlatBuffers
+type, not ObjectBox's semantic annotation on top) — see "Beyond `flatc`"
+below.
+
+Decoding those raw bytes into typed objects is left to the official
+`flatc --python` compiler — not this package, and not [`ob-dump`](..)'s own
+C++ core either. This is a deliberate toolkit split, not a missing feature:
+see the parent project's [`docs/BACKLOG.md`](../docs/BACKLOG.md#schema-export---schema-and---fbs)
+for the reasoning.
+
+## Workflow
+
+```sh
+# 1. Generate a schema JSON (entityId -> table name/shape) and a .fbs from
+#    your ObjectBox model, using the ob_dump CLI (see ../README.md):
+ob_dump --schema objectbox-model.json -o schema.json
+ob_dump --fbs    objectbox-model.json -o schema.fbs
+
+# 2. Generate typed Python classes with the *official* FlatBuffers compiler.
+flatc --python -o models/ schema.fbs
+```
+
+```python
+import ob_dump_reader as ob
+from models.Ammo import Ammo  # from flatc --python, see step 2 above
+
+AMMO_ENTITY_ID = 1  # from schema.json
+
+
+def on_record(record: ob.ObRecord) -> None:
+    if record.entity_id == AMMO_ENTITY_ID:
+        ammo = Ammo.GetRootAs(record.data)  # flatc-generated class decodes the bytes
+        print(f"{ammo.Name()}: {ammo.BcG1()}")
+        # ... insert into whatever your new database is.
+
+
+ob.read_objectbox_records("/path/to/objectbox/dir", on_record)
+```
+
+## Async
+
+`ob_dump_reader.aio` mirrors the same API as `async`/`await` coroutines
+(built on [py-lmdb's own `lmdb.aio`](https://py-lmdb.readthedocs.io/en/latest/)
+executor-based wrapper — LMDB itself has no native async I/O, so this
+dispatches the same blocking calls to a thread executor rather than
+avoiding them):
+
+```python
+import ob_dump_reader.aio as ob
+
+
+async def on_record(record: ob.ObRecord) -> None:
+    ...
+
+
+await ob.read_objectbox_records("/path/to/objectbox/dir", on_record)
+```
+
+## Reads are in place, no copy
+
+`read_objectbox_records`/`read_objectbox_to_many_targets` read
+`data.mdb`/`lock.mdb` directly, with no temporary copy step — they open
+the LMDB environment itself `readonly=True` (matching `ob-dump`'s own C++
+`LmdbReader`), so they never need a write-capable handle. This is exactly
+what LMDB's MVCC design is for: any number of readers can safely run
+alongside one concurrent writer, in any process, with zero risk to the
+original data — even while a live ObjectBox process has the same store
+open.
+
+## Beyond `flatc`: ToMany relations and Flex/ExternalPropertyType fields
+
+`ob_dump --schema` lists each property's `externalType` and each entity's
+`relations` (id, name, target entity) when present — use that to know
+which of your fields need one of these:
+
+```python
+import ob_dump_reader as ob
+from ob_dump_reader.decode_helpers import (
+    decode_flex,
+    bytes_to_hex,
+    bytes_to_uuid_string,
+    try_parse_json_string,
+)
+
+# ToMany: not part of the FlatBuffers table at all, so flatc has no
+# accessor for it — relation_id/source_object_id come from `ob_dump --schema`
+# and the record you're looking at (record.object_id).
+author_ids = ob.read_objectbox_to_many_targets(db_dir, relation_id, record.object_id)
+
+# Flex: flatc gives you the raw bytes (a bytes/bytearray field) —
+# decode with decode_flex.
+value = decode_flex(ammo.SomeFlexField())
+
+# ExternalPropertyType: flatc gives you the base type's plain value
+# (a byte blob for Uuid/Int128/Decimal128/Bson, a string for
+# Json/JavaScript/JsonToNative) — decode with the matching helper.
+uuid = bytes_to_uuid_string(ammo.SomeUuidField())
+blob = bytes_to_hex(ammo.SomeBsonField())
+parsed = try_parse_json_string(ammo.SomeJsonField())
+```
+
+Every one of these mirrors `ob-dump`'s own C++ decode (`src/fb_decode.cpp`)
+exactly — same hex/UUID formatting, same JSON-parse-with-string-fallback
+for `JavaScript`, same forward-only relation direction (see
+`docs/BACKLOG.md` "ToMany relations" for why: the backward direction is an
+auto-maintained index for ObjectBox's own query engine, not needed for a
+one-directional dump).
+
+## Why this shape
+
+The alternative — an FFI wrapper around `ob-dump`'s C++ core — would mean
+building/vendoring native C++ from a PyPI package for every platform. Not
+needed here: [`py-lmdb`](https://github.com/jnwatson/py-lmdb) already gives
+a mature, actively-maintained Python binding to LMDB, and `flatc --python`
+already gives an officially generated, correct decoder. This package is
+only the small piece connecting the two — LMDB traversal and the
+ObjectBox key format (`docs/BACKLOG.md` in the parent project) — and stays
+that small on purpose.
+
+## Integrity & Licensing
+
+`ob-dump` was developed as an independent implementation for reading data stored in the ObjectBox format. It adheres to a "Clean Room Design" approach regarding binary software:
+- **Purpose-Limited:** Built solely to support data recovery and migration to another database, for projects whose own license is incompatible with `objectbox-c`'s (a closed-source binary — see "Why this exists" in the parent project's [`docs/BACKLOG.md`](../docs/BACKLOG.md)). Not intended as, and not pursued as, a competing product to ObjectBox itself — no write support, no query engine, no ongoing-database use case, strictly a one-time read-only export path out of an existing store.
+- **No Reverse Engineering:** We have performed no decompilation, disassembly, or any other analysis of the closed-source `objectbox-c` binary.
+- **Open Specification:** Data parsing is based exclusively on public formats (LMDB and FlatBuffers) and the open-source code of the official schema generator ([`objectbox_generator`](https://github.com/objectbox/objectbox-dart/tree/main/generator), licensed under Apache 2.0).
+- **Model-Driven:** The decoding process is driven by the user-provided `objectbox-model.json` file, which is an open, user-accessible schema definition.
+
+This approach ensures full licensing integrity: `ob-dump` is an independent software project that contains no proprietary or misappropriated code, making it suitable for integration into projects with any licensing requirements.
+
+## License
+
+MIT — see [`LICENSE`](LICENSE).

ob_dump_reader-0.0.0.dev0.dist-info/RECORD

@@ -0,0 +1,13 @@
+ob_dump_reader/__init__.py,sha256=xZo6JxEXBEyael1cjrdEdLQFX6n89yvfCrwN1MqA8Vo,2914
+ob_dump_reader/__main__.py,sha256=9YXM28A2mbVlhj6dy80KFRx0oGCD_uGcxVhP4klP9mU,841
+ob_dump_reader/_constants.py,sha256=oCFwqNMfDc1OS1Ow4Rpr8VoFzuvs-qaznJdwytJUwXc,129
+ob_dump_reader/aio.py,sha256=E4m_AJjdxNjW-hjvFcmU9gTWjPo6TE796nscBygsAak,3182
+ob_dump_reader/decode_helpers.py,sha256=MSEzI1PMtTEU7__6akrBIswBQCgkjEXC7j3Mtav1L8k,2493
+ob_dump_reader/__pycache__/__init__.cpython-314.pyc,sha256=Vag19DQHJHiF8Ca_Oyz9oyfcqU6tWMTCJVeKIqRu72U,5199
+ob_dump_reader/__pycache__/__main__.cpython-314.pyc,sha256=m_3zu3mF08ZAsYoGLaav6E_b0w4FmiX_IEItUcnqju8,517
+ob_dump_reader-0.0.0.dev0.dist-info/licenses/LICENSE,sha256=B0rOBJOtqUXUOlNfEGwYAreh9Ch-S3LcsQ9PgRToFbc,1085
+ob_dump_reader-0.0.0.dev0.dist-info/METADATA,sha256=KEXqhdRVsNV9TcTD69grJGOUBTcJUmlmmw5IfcAUKp8,8043
+ob_dump_reader-0.0.0.dev0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+ob_dump_reader-0.0.0.dev0.dist-info/entry_points.txt,sha256=auDiE-9V3YNFv8-TU42PlJnqVSZOuwDJyjz5_E7Y0b4,64
+ob_dump_reader-0.0.0.dev0.dist-info/top_level.txt,sha256=69BfNbTE2kG0migwG2gFnv7vb0qjG-e9vIS0CZfkvTU,15
+ob_dump_reader-0.0.0.dev0.dist-info/RECORD,,

ob_dump_reader-0.0.0.dev0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (83.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

ob_dump_reader-0.0.0.dev0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+ob-dump-reader = ob_dump_reader.__main__:main

ob_dump_reader-0.0.0.dev0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Dmytro Yaroshenko (o-murphy)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ob_dump_reader-0.0.0.dev0.dist-info/top_level.txt

@@ -0,0 +1 @@
+ob_dump_reader