orbynt-protocol 0.1.0__py3-none-any.whl

omp/__init__.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+from omp.core.reader import read
+from omp.core.writer import write
+from omp.exceptions import InvalidOrbFile, OMPError, UnsupportedFeature, VersionMismatch
+from omp.inject import inject
+from omp.inspect import inspect
+from omp.merge import merge
+from omp.protos import (
+    export_proto_bundle,
+    get_ai_schema_context,
+    get_all_proto_sources,
+    get_proto_bundle,
+    get_proto_source,
+    list_proto_files,
+)
+from omp.query import MemoryMatch, query
+from omp.schema.convert import export_passport as export
+from omp.schema.version import PACKAGE_VERSION, PROTOCOL_VERSION
+
+__version__ = PACKAGE_VERSION
+
+__all__ = [
+    "InvalidOrbFile",
+    "MemoryMatch",
+    "OMPError",
+    "PROTOCOL_VERSION",
+    "UnsupportedFeature",
+    "VersionMismatch",
+    "__version__",
+    "export",
+    "export_proto_bundle",
+    "get_ai_schema_context",
+    "get_all_proto_sources",
+    "get_proto_bundle",
+    "get_proto_source",
+    "inject",
+    "inspect",
+    "list_proto_files",
+    "merge",
+    "query",
+    "read",
+    "write",
+]

omp/cli/__init__.py

@@ -0,0 +1,2 @@
+"""Command line interface for OMP."""
+

omp/cli/main.py

@@ -0,0 +1,210 @@
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+import omp
+from omp.exceptions import OMPError
+from omp.schema.convert import to_json, to_markdown
+from omp.schema.version import PACKAGE_VERSION, PROTOCOL_VERSION
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = _build_parser()
+    args = parser.parse_args(argv)
+    try:
+        return args.func(args)
+    except OMPError as exc:
+        print(f"omp: {exc}", file=sys.stderr)
+        return 1
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="omp")
+    sub = parser.add_subparsers(required=True)
+
+    write = sub.add_parser("write", help="create a .orb file")
+    write.add_argument("--name", default="", help="identity display name")
+    write.add_argument("--out", required=True, help="output .orb path")
+    write.add_argument("--memory", action="append", default=[], help="kind:fact|summary:...|tags:a,b")
+    write.add_argument("--pref", action="append", default=[], help="key=value")
+    write.add_argument("--compress", action="store_true", help="zstd-compress chunks")
+    write.set_defaults(func=_cmd_write)
+
+    read = sub.add_parser("read", help="read a .orb file")
+    read.add_argument("path")
+    read.add_argument("--format", choices=["json", "text", "markdown"], default="text")
+    read.add_argument("--out")
+    read.set_defaults(func=_cmd_read)
+
+    inspect = sub.add_parser("inspect", help="show a human-readable summary")
+    inspect.add_argument("path")
+    inspect.set_defaults(func=_cmd_inspect)
+
+    query = sub.add_parser("query", help="query memories")
+    query.add_argument("path")
+    query.add_argument("--text", default="")
+    query.add_argument("--kind")
+    query.add_argument("--tag", action="append", default=[])
+    query.add_argument("--limit", type=int, default=5)
+    query.set_defaults(func=_cmd_query)
+
+    merge = sub.add_parser("merge", help="merge two passports")
+    merge.add_argument("a")
+    merge.add_argument("b")
+    merge.add_argument("--out", required=True)
+    merge.add_argument(
+        "--strategy",
+        choices=["latest", "union", "source_a", "source_b"],
+        default="latest",
+    )
+    merge.set_defaults(func=_cmd_merge)
+
+    export = sub.add_parser("export", help="export a .orb file")
+    export.add_argument("path")
+    export.add_argument("--format", choices=["json", "markdown"], required=True)
+    export.add_argument("--out", required=True)
+    export.set_defaults(func=_cmd_export)
+
+    proto = sub.add_parser("proto", help="inspect bundled protobuf schemas")
+    proto_sub = proto.add_subparsers(required=True)
+    proto_list = proto_sub.add_parser("list", help="list bundled .proto files")
+    proto_list.set_defaults(func=_cmd_proto_list)
+    proto_show = proto_sub.add_parser("show", help="print one bundled .proto file")
+    proto_show.add_argument("name")
+    proto_show.set_defaults(func=_cmd_proto_show)
+    proto_bundle = proto_sub.add_parser("bundle", help="print or write all bundled .proto files")
+    proto_bundle.add_argument("--out")
+    proto_bundle.set_defaults(func=_cmd_proto_bundle)
+
+    verify = sub.add_parser("verify", help="verify checksum and open-tier status")
+    verify.add_argument("path")
+    verify.set_defaults(func=_cmd_verify)
+
+    version = sub.add_parser("version", help="show package and protocol versions")
+    version.set_defaults(func=_cmd_version)
+    return parser
+
+
+def _cmd_write(args: argparse.Namespace) -> int:
+    memories = [_parse_memory(value) for value in args.memory]
+    preferences = dict(_parse_pref(value) for value in args.pref)
+    omp.write(
+        args.out,
+        identity={"display_name": args.name},
+        memories=memories,
+        preferences=preferences,
+        compress=args.compress,
+    )
+    print(f"wrote {args.out}")
+    return 0
+
+
+def _cmd_read(args: argparse.Namespace) -> int:
+    passport = omp.read(args.path)
+    if args.format == "json":
+        output = to_json(passport)
+    elif args.format == "markdown":
+        output = to_markdown(passport)
+    else:
+        output = omp.inject(passport, target="generic")
+    _write_or_print(output, args.out)
+    return 0
+
+
+def _cmd_inspect(args: argparse.Namespace) -> int:
+    omp.inspect(args.path)
+    return 0
+
+
+def _cmd_query(args: argparse.Namespace) -> int:
+    passport = omp.read(args.path)
+    matches = omp.query(
+        passport,
+        text=args.text,
+        kind=args.kind,
+        tags=args.tag,
+        limit=args.limit,
+    )
+    for match in matches:
+        print(f"{match.score:.3f}\t{match.memory.summary}")
+    return 0
+
+
+def _cmd_merge(args: argparse.Namespace) -> int:
+    merged = omp.merge(omp.read(args.a), omp.read(args.b), strategy=args.strategy)
+    omp.write(args.out, passport=merged)
+    print(f"wrote {args.out}")
+    return 0
+
+
+def _cmd_export(args: argparse.Namespace) -> int:
+    passport = omp.read(args.path)
+    omp.export(passport, format=args.format, path=args.out)
+    print(f"wrote {args.out}")
+    return 0
+
+
+def _cmd_proto_list(args: argparse.Namespace) -> int:
+    for name in omp.list_proto_files():
+        print(name)
+    return 0
+
+
+def _cmd_proto_show(args: argparse.Namespace) -> int:
+    print(omp.get_proto_source(args.name), end="")
+    return 0
+
+
+def _cmd_proto_bundle(args: argparse.Namespace) -> int:
+    output = omp.get_proto_bundle()
+    _write_or_print(output, args.out)
+    return 0
+
+
+def _cmd_verify(args: argparse.Namespace) -> int:
+    passport = omp.read(args.path)
+    print("checksum valid")
+    print("signature none (open tier)")
+    print(f"verified {passport.verified}")
+    return 0
+
+
+def _cmd_version(args: argparse.Namespace) -> int:
+    print(f"omp package {PACKAGE_VERSION}")
+    print(f"protocol {PROTOCOL_VERSION}")
+    return 0
+
+
+def _parse_memory(value: str) -> dict:
+    result: dict[str, object] = {}
+    for part in value.split("|"):
+        if not part:
+            continue
+        key, sep, raw = part.partition(":")
+        if not sep:
+            raise OMPError(f"invalid --memory segment: {part}")
+        if key == "tags":
+            result[key] = [tag.strip() for tag in raw.split(",") if tag.strip()]
+        else:
+            result[key] = raw
+    return result
+
+
+def _parse_pref(value: str) -> tuple[str, str]:
+    key, sep, raw = value.partition("=")
+    if not sep:
+        raise OMPError(f"invalid --pref value: {value}")
+    return key, raw
+
+
+def _write_or_print(output: str, path: str | None) -> None:
+    if path:
+        Path(path).write_text(output, encoding="utf-8")
+    else:
+        print(output)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

omp/core/__init__.py

@@ -0,0 +1,26 @@
+from omp.core.framer import (
+    Chunk,
+    ChunkEntry,
+    ChunkFlag,
+    ChunkType,
+    FileFlag,
+    FramedOrb,
+    frame_chunks,
+    parse_framed,
+    read_framed_file,
+    write_framed_file,
+)
+
+__all__ = [
+    "Chunk",
+    "ChunkEntry",
+    "ChunkFlag",
+    "ChunkType",
+    "FileFlag",
+    "FramedOrb",
+    "frame_chunks",
+    "parse_framed",
+    "read_framed_file",
+    "write_framed_file",
+]
+

omp/core/checksum.py

@@ -0,0 +1,12 @@
+from __future__ import annotations
+
+import hashlib
+
+
+def sha256(data: bytes) -> bytes:
+    return hashlib.sha256(data).digest()
+
+
+def sha256_hex(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+

omp/core/compressor.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from omp.exceptions import UnsupportedFeature
+
+
+def _zstd():
+    try:
+        import zstandard as zstd
+    except ImportError as exc:
+        raise UnsupportedFeature(
+            "zstandard is required for compressed .orb chunks. Install omp with zstandard."
+        ) from exc
+    return zstd
+
+
+def compress(data: bytes) -> bytes:
+    zstd = _zstd()
+    return zstd.ZstdCompressor().compress(data)
+
+
+def decompress(data: bytes) -> bytes:
+    zstd = _zstd()
+    return zstd.ZstdDecompressor().decompress(data)
+

omp/core/framer.py

@@ -0,0 +1,278 @@
+from __future__ import annotations
+
+import struct
+from dataclasses import dataclass
+from enum import IntEnum, IntFlag
+from pathlib import Path
+from typing import Iterable
+
+from omp.core.checksum import sha256
+from omp.exceptions import InvalidOrbFile, VersionMismatch
+from omp.schema.version import PROTOCOL_MAJOR
+
+MAGIC = b"ORB1"
+FOOTER_MAGIC = b"ORBF"
+
+HEADER_STRUCT = struct.Struct("<4sHHHHII")
+CHUNK_ENTRY_STRUCT = struct.Struct("<IIQQ")
+CHUNK_PREAMBLE_STRUCT = struct.Struct("<IQ")
+FOOTER_STRUCT = struct.Struct("<Q32s4s")
+
+HEADER_FIXED_LENGTH = HEADER_STRUCT.size
+CHUNK_ENTRY_LENGTH = CHUNK_ENTRY_STRUCT.size
+CHUNK_PREAMBLE_LENGTH = CHUNK_PREAMBLE_STRUCT.size
+FOOTER_LENGTH = FOOTER_STRUCT.size
+
+
+class FileFlag(IntFlag):
+    HAS_COMPRESSION = 1 << 0
+    HAS_ENCRYPTION = 1 << 1
+    HAS_SIGNATURE = 1 << 2
+    HAS_GRAPH = 1 << 3
+    HAS_EMBEDDINGS = 1 << 4
+    HAS_ADAPTERS = 1 << 5
+
+
+class ChunkFlag(IntFlag):
+    COMPRESSED = 1 << 0
+    ENCRYPTED = 1 << 1
+    OPTIONAL = 1 << 2
+
+
+class ChunkType(IntEnum):
+    PROTOCOL_METADATA = 1
+    IDENTITY = 2
+    MEMORY_BUNDLE = 3
+    MEMORY_GRAPH = 4
+    EMBEDDING_INDEX = 5
+    ADAPTER_METADATA = 6
+    REGISTRY_SNAPSHOT = 7
+    EXPORT_POLICY = 8
+    PROFILE = 9
+    PREFERENCES = 10
+    GOALS = 11
+    VALUES = 12
+    SKILLS = 13
+    INTERESTS = 14
+    RELATIONSHIPS = 15
+    PROVENANCE = 16
+    INTEGRITY_METADATA = 17
+    CUSTOM_EXTENSION = 1000
+
+
+KNOWN_CHUNK_TYPES = {item.value for item in ChunkType}
+OPEN_FORBIDDEN_FILE_FLAGS = FileFlag.HAS_ENCRYPTION | FileFlag.HAS_SIGNATURE
+OPEN_FORBIDDEN_CHUNK_FLAGS = ChunkFlag.ENCRYPTED
+RESERVED_FILE_FLAG_MASK = ~sum(flag.value for flag in FileFlag)
+RESERVED_CHUNK_FLAG_MASK = ~sum(flag.value for flag in ChunkFlag)
+
+
+@dataclass(frozen=True)
+class Chunk:
+    chunk_type: int
+    data: bytes
+    flags: int = 0
+
+
+@dataclass(frozen=True)
+class ChunkEntry:
+    chunk_type: int
+    chunk_flags: int
+    chunk_offset: int
+    chunk_length: int
+
+
+@dataclass(frozen=True)
+class FramedOrb:
+    version: tuple[int, int, int]
+    flags: int
+    chunks: list[Chunk]
+    entries: list[ChunkEntry]
+    checksum: bytes
+
+
+def frame_chunks(
+    chunks: Iterable[Chunk],
+    *,
+    version: tuple[int, int, int] = (1, 0, 0),
+) -> bytes:
+    chunk_list = list(chunks)
+    flags = _derive_file_flags(chunk_list)
+    header_length = HEADER_FIXED_LENGTH + len(chunk_list) * CHUNK_ENTRY_LENGTH
+
+    entries: list[ChunkEntry] = []
+    records: list[bytes] = []
+    offset = header_length
+    for chunk in chunk_list:
+        _validate_chunk_for_write(chunk)
+        record = CHUNK_PREAMBLE_STRUCT.pack(chunk.chunk_type, len(chunk.data)) + chunk.data
+        entries.append(
+            ChunkEntry(
+                chunk_type=chunk.chunk_type,
+                chunk_flags=chunk.flags,
+                chunk_offset=offset,
+                chunk_length=len(chunk.data),
+            )
+        )
+        records.append(record)
+        offset += len(record)
+
+    major, minor, patch = version
+    header = HEADER_STRUCT.pack(
+        MAGIC,
+        major,
+        minor,
+        patch,
+        flags,
+        header_length,
+        len(chunk_list),
+    )
+    table = b"".join(
+        CHUNK_ENTRY_STRUCT.pack(
+            entry.chunk_type,
+            entry.chunk_flags,
+            entry.chunk_offset,
+            entry.chunk_length,
+        )
+        for entry in entries
+    )
+    body = header + table + b"".join(records)
+    footer = FOOTER_STRUCT.pack(len(body) + FOOTER_LENGTH, sha256(body), FOOTER_MAGIC)
+    return body + footer
+
+
+def parse_framed(data: bytes) -> FramedOrb:
+    if len(data) < HEADER_FIXED_LENGTH + FOOTER_LENGTH:
+        raise InvalidOrbFile("file is too small to be a valid .orb container")
+
+    magic, major, minor, patch, flags, header_length, chunk_count = HEADER_STRUCT.unpack_from(
+        data, 0
+    )
+    if magic != MAGIC:
+        raise InvalidOrbFile("invalid .orb magic bytes")
+    if major > PROTOCOL_MAJOR:
+        raise VersionMismatch(f"unsupported OMP major version {major}")
+    _validate_file_flags(flags)
+
+    minimum_header_length = HEADER_FIXED_LENGTH + chunk_count * CHUNK_ENTRY_LENGTH
+    if header_length < minimum_header_length:
+        raise InvalidOrbFile("header_length is shorter than the chunk table")
+    if header_length > len(data) - FOOTER_LENGTH:
+        raise InvalidOrbFile("header_length points beyond the file body")
+
+    footer_start = len(data) - FOOTER_LENGTH
+    total_length, digest, footer_magic = FOOTER_STRUCT.unpack_from(data, footer_start)
+    if footer_magic != FOOTER_MAGIC:
+        raise InvalidOrbFile("invalid .orb footer magic")
+    if total_length != len(data):
+        raise InvalidOrbFile("footer total_file_length does not match actual file size")
+    if sha256(data[:footer_start]) != digest:
+        raise InvalidOrbFile("footer SHA-256 checksum mismatch")
+
+    entries = _parse_entries(data, chunk_count)
+    chunks: list[Chunk] = []
+    for entry in entries:
+        _validate_entry(entry, footer_start, header_length)
+        if entry.chunk_type not in KNOWN_CHUNK_TYPES:
+            if entry.chunk_flags & ChunkFlag.OPTIONAL:
+                continue
+            raise VersionMismatch(f"unknown required chunk type {entry.chunk_type}")
+
+        actual_type, actual_length = CHUNK_PREAMBLE_STRUCT.unpack_from(data, entry.chunk_offset)
+        if actual_type != entry.chunk_type:
+            raise InvalidOrbFile("chunk table type does not match chunk payload type")
+        if actual_length != entry.chunk_length:
+            raise InvalidOrbFile("chunk table length does not match chunk payload length")
+
+        payload_start = entry.chunk_offset + CHUNK_PREAMBLE_LENGTH
+        payload_end = payload_start + entry.chunk_length
+        chunks.append(
+            Chunk(
+                chunk_type=entry.chunk_type,
+                flags=entry.chunk_flags,
+                data=data[payload_start:payload_end],
+            )
+        )
+
+    return FramedOrb(
+        version=(major, minor, patch),
+        flags=flags,
+        chunks=chunks,
+        entries=entries,
+        checksum=digest,
+    )
+
+
+def write_framed_file(
+    path: str | Path,
+    chunks: Iterable[Chunk],
+    *,
+    version: tuple[int, int, int] = (1, 0, 0),
+) -> bytes:
+    data = frame_chunks(chunks, version=version)
+    Path(path).write_bytes(data)
+    return data
+
+
+def read_framed_file(path: str | Path) -> FramedOrb:
+    return parse_framed(Path(path).read_bytes())
+
+
+def _derive_file_flags(chunks: list[Chunk]) -> int:
+    flags = FileFlag(0)
+    for chunk in chunks:
+        if chunk.flags & ChunkFlag.COMPRESSED:
+            flags |= FileFlag.HAS_COMPRESSION
+        if chunk.flags & ChunkFlag.ENCRYPTED:
+            flags |= FileFlag.HAS_ENCRYPTION
+        if chunk.chunk_type == ChunkType.MEMORY_GRAPH:
+            flags |= FileFlag.HAS_GRAPH
+        if chunk.chunk_type == ChunkType.EMBEDDING_INDEX:
+            flags |= FileFlag.HAS_EMBEDDINGS
+        if chunk.chunk_type == ChunkType.ADAPTER_METADATA:
+            flags |= FileFlag.HAS_ADAPTERS
+    _validate_file_flags(int(flags))
+    return int(flags)
+
+
+def _validate_chunk_for_write(chunk: Chunk) -> None:
+    if chunk.flags & RESERVED_CHUNK_FLAG_MASK:
+        raise InvalidOrbFile(f"reserved chunk flags are set: {chunk.flags}")
+    if chunk.flags & OPEN_FORBIDDEN_CHUNK_FLAGS:
+        raise InvalidOrbFile("open-tier .orb files cannot contain encrypted chunks")
+    if chunk.chunk_type == ChunkType.CUSTOM_EXTENSION and not (
+        chunk.flags & ChunkFlag.OPTIONAL
+    ):
+        raise InvalidOrbFile("CUSTOM_EXTENSION chunks must be flagged OPTIONAL")
+
+
+def _validate_file_flags(flags: int) -> None:
+    if flags & RESERVED_FILE_FLAG_MASK:
+        raise InvalidOrbFile(f"reserved file flags are set: {flags}")
+    if flags & OPEN_FORBIDDEN_FILE_FLAGS:
+        raise InvalidOrbFile("open-tier .orb files cannot set encryption/signature flags")
+
+
+def _parse_entries(data: bytes, chunk_count: int) -> list[ChunkEntry]:
+    entries: list[ChunkEntry] = []
+    offset = HEADER_FIXED_LENGTH
+    for _ in range(chunk_count):
+        chunk_type, chunk_flags, chunk_offset, chunk_length = CHUNK_ENTRY_STRUCT.unpack_from(
+            data, offset
+        )
+        if chunk_flags & RESERVED_CHUNK_FLAG_MASK:
+            raise InvalidOrbFile(f"reserved chunk flags are set: {chunk_flags}")
+        if chunk_flags & OPEN_FORBIDDEN_CHUNK_FLAGS:
+            raise InvalidOrbFile("open-tier .orb files cannot contain encrypted chunks")
+        entries.append(ChunkEntry(chunk_type, chunk_flags, chunk_offset, chunk_length))
+        offset += CHUNK_ENTRY_LENGTH
+    return entries
+
+
+def _validate_entry(entry: ChunkEntry, footer_start: int, header_length: int) -> None:
+    if entry.chunk_offset < header_length:
+        raise InvalidOrbFile("chunk offset points inside the header")
+    payload_start = entry.chunk_offset + CHUNK_PREAMBLE_LENGTH
+    payload_end = payload_start + entry.chunk_length
+    if payload_start > footer_start or payload_end > footer_start:
+        raise InvalidOrbFile("chunk extends beyond the file body")