deltagraphar 0.1.0__py3-none-any.whl

deltagraphar/cli.py

@@ -0,0 +1,140 @@
+"""Command-line interface for DeltaGraphAr."""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+
+def _get_backend(args):
+    from deltagraphar.versioning.local_backend import LocalBackend
+    return LocalBackend(args.repo)
+
+
+def cmd_log(args):
+    b = _get_backend(args)
+    commits = b.log()
+    if not commits:
+        print("(no commits)")
+        return
+    for c in commits:
+        from datetime import datetime, timezone
+        dt = datetime.fromtimestamp(c.timestamp, tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
+        meta = f"  {c.metadata}" if c.metadata else ""
+        print(f"{c.ref[:8]}  {dt}  {c.message}{meta}")
+
+
+def cmd_tag(args):
+    b = _get_backend(args)
+    log = b.log()
+    if not log:
+        print("error: no commits to tag", file=sys.stderr)
+        sys.exit(1)
+    ref = log[-1].ref
+    b.tag(args.name, ref)
+    print(f"tagged {ref[:8]} as {args.name!r}")
+
+
+def cmd_neighbors(args):
+    from deltagraphar.versioning.local_backend import LocalBackend
+    from deltagraphar.store.graphstore import GraphStore
+    from deltagraphar.format.reader import read_yaml
+    from deltagraphar.format.paths import vertex_yaml_path, edge_yaml_path
+    from deltagraphar.format.schema import GraphInfo, VertexInfo, EdgeInfo
+
+    b = LocalBackend(args.repo)
+    ref = args.ref or None
+
+    etype = tuple(args.etype.split(","))
+    if len(etype) != 3:
+        print("error: --etype must be 'src,edge,dst'", file=sys.stderr)
+        sys.exit(1)
+
+    src_label, et, dst_label = etype
+    vi_data = read_yaml(b, vertex_yaml_path(args.label), ref=ref)
+    ei_data = read_yaml(b, edge_yaml_path(src_label, et, dst_label), ref=ref)
+
+    vi = VertexInfo(label=vi_data["label"], chunk_size=vi_data["chunk_size"])
+    ei = EdgeInfo(
+        src_type=ei_data["src_type"],
+        edge_type=ei_data["edge_type"],
+        dst_type=ei_data["dst_type"],
+        chunk_size=ei_data["chunk_size"],
+        src_chunk_size=ei_data["src_chunk_size"],
+    )
+    gi = GraphInfo(name="graph", prefix="", vertex_infos=[vi], edge_infos=[ei])
+    gs = GraphStore(b, gi, vertex_chunk_size=vi_data["chunk_size"])
+    nbrs = gs.out_neighbors(args.label, args.vertex, etype, ref=ref)
+    print(json.dumps(nbrs))
+
+
+def cmd_compact(args):
+    from deltagraphar.versioning.local_backend import LocalBackend
+    from deltagraphar.store.graphstore import GraphStore
+    from deltagraphar.format.reader import read_yaml
+    from deltagraphar.format.paths import vertex_yaml_path, edge_yaml_path
+    from deltagraphar.format.schema import GraphInfo, VertexInfo, EdgeInfo
+
+    b = LocalBackend(args.repo)
+    etype = tuple(args.etype.split(","))
+    if len(etype) != 3:
+        print("error: --etype must be 'src,edge,dst'", file=sys.stderr)
+        sys.exit(1)
+
+    src_label, et, dst_label = etype
+    vi_data = read_yaml(b, vertex_yaml_path(src_label))
+    ei_data = read_yaml(b, edge_yaml_path(src_label, et, dst_label))
+
+    vi = VertexInfo(label=vi_data["label"], chunk_size=vi_data["chunk_size"])
+    ei = EdgeInfo(
+        src_type=ei_data["src_type"],
+        edge_type=ei_data["edge_type"],
+        dst_type=ei_data["dst_type"],
+        chunk_size=ei_data["chunk_size"],
+        src_chunk_size=ei_data["src_chunk_size"],
+    )
+    gi = GraphInfo(name="graph", prefix="", vertex_infos=[vi], edge_infos=[ei])
+    gs = GraphStore(b, gi, vertex_chunk_size=vi_data["chunk_size"])
+
+    vchunks = [int(v) for v in args.vchunks.split(",")] if args.vchunks else None
+    ref = gs.compact(etype, vchunks=vchunks)
+    print(f"compacted → {ref[:8]}")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog="deltagraphar",
+        description="DeltaGraphAr — versioned property-graph store",
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    p_log = sub.add_parser("log", help="Show commit history")
+    p_log.add_argument("--repo", required=True, help="Path to local repo")
+    p_log.set_defaults(func=cmd_log)
+
+    p_tag = sub.add_parser("tag", help="Tag the latest commit")
+    p_tag.add_argument("--repo", required=True)
+    p_tag.add_argument("name", help="Tag name")
+    p_tag.set_defaults(func=cmd_tag)
+
+    p_nbr = sub.add_parser("neighbors", help="List out-neighbors of a vertex at a ref")
+    p_nbr.add_argument("--repo", required=True)
+    p_nbr.add_argument("--label", required=True, help="Vertex label")
+    p_nbr.add_argument("--vertex", required=True, help="Logical vertex ID")
+    p_nbr.add_argument("--etype", required=True, help="Edge type as 'src,edge,dst'")
+    p_nbr.add_argument("--ref", default=None, help="Commit ref or tag (default: HEAD)")
+    p_nbr.set_defaults(func=cmd_neighbors)
+
+    p_compact = sub.add_parser("compact", help="Compact delta into ordered CSR")
+    p_compact.add_argument("--repo", required=True)
+    p_compact.add_argument("--etype", required=True, help="Edge type as 'src,edge,dst'")
+    p_compact.add_argument("--vchunks", default=None,
+                           help="Comma-separated vchunk indices (default: auto-discover)")
+    p_compact.set_defaults(func=cmd_compact)
+
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()

deltagraphar/format/paths.py

@@ -0,0 +1,35 @@
+def vertex_chunk_path(label: str, pg_prefix: str, chunk_idx: int) -> str:
+    return f"vertex/{label}/{pg_prefix}/chunk{chunk_idx}"
+
+
+def vid_map_chunk_path(label: str, chunk_idx: int) -> str:
+    return f"vertex/{label}/__vid_map__/chunk{chunk_idx}"
+
+
+def adj_list_chunk_path(
+    src: str, etype: str, dst: str, adj_type: str, vchunk: int, chunk_idx: int
+) -> str:
+    return f"edge/{src}_{etype}_{dst}/{adj_type}/adj_list/part{vchunk}/chunk{chunk_idx}"
+
+
+def offset_chunk_path(src: str, etype: str, dst: str, vchunk: int) -> str:
+    return f"edge/{src}_{etype}_{dst}/ordered_by_source/offset/part{vchunk}/chunk0"
+
+
+def edge_prop_chunk_path(
+    src: str, etype: str, dst: str,
+    adj_type: str, pg_prefix: str, vchunk: int, chunk_idx: int,
+) -> str:
+    return f"edge/{src}_{etype}_{dst}/{adj_type}/{pg_prefix}/part{vchunk}/chunk{chunk_idx}"
+
+
+def graph_yaml_path(name: str) -> str:
+    return f"{name}.graph.yml"
+
+
+def vertex_yaml_path(label: str) -> str:
+    return f"{label}.vertex.yml"
+
+
+def edge_yaml_path(src: str, etype: str, dst: str) -> str:
+    return f"{src}_{etype}_{dst}.edge.yml"

deltagraphar/format/reader.py

@@ -0,0 +1,58 @@
+import io
+import yaml
+import pyarrow as pa
+import pyarrow.parquet as pq
+import pyarrow.compute as pc
+
+from deltagraphar.format.paths import offset_chunk_path
+
+
+def read_table(backend, path: str, ref=None) -> pa.Table:
+    data = backend.read_file(path, ref=ref)
+    return pq.read_table(io.BytesIO(data))
+
+
+def read_yaml(backend, path: str, ref=None) -> dict:
+    return yaml.safe_load(backend.read_file(path, ref=ref))
+
+
+def read_adj_list(
+    backend, src: str, etype: str, dst: str, adj_type: str, vchunk: int, ref=None
+) -> pa.Table:
+    """Concatenate all chunk files for one (adj_type, vchunk) partition."""
+    prefix = f"edge/{src}_{etype}_{dst}/{adj_type}/adj_list/part{vchunk}"
+    paths = sorted(backend.list(prefix, ref=ref))
+    if not paths:
+        return pa.table({
+            "src_physical": pa.array([], type=pa.int64()),
+            "dst_physical": pa.array([], type=pa.int64()),
+        })
+    return pa.concat_tables([read_table(backend, p, ref=ref) for p in paths])
+
+
+def read_offsets(backend, src: str, etype: str, dst: str, vchunk: int, ref=None) -> list[int]:
+    """Read the CSR offset array for a vchunk; returns [] if not yet compacted."""
+    path = offset_chunk_path(src, etype, dst, vchunk)
+    try:
+        tbl = read_table(backend, path, ref=ref)
+        return tbl["offset"].to_pylist()
+    except FileNotFoundError:
+        return []
+
+
+def scan_delta(
+    backend, src: str, etype: str, dst: str, vchunk: int, src_physical: int, ref=None
+) -> list[int]:
+    """Return dst_physical list from unordered delta where src_physical matches."""
+    delta = read_adj_list(backend, src, etype, dst, "unordered_by_source", vchunk, ref=ref)
+    if len(delta) == 0:
+        return []
+    mask = pc.equal(delta["src_physical"], src_physical)
+    return delta.filter(mask)["dst_physical"].to_pylist()
+
+
+def count_rows(
+    backend, src: str, etype: str, dst: str, adj_type: str, vchunk: int, ref=None
+) -> int:
+    tbl = read_adj_list(backend, src, etype, dst, adj_type, vchunk, ref=ref)
+    return len(tbl)

deltagraphar/format/schema.py

@@ -0,0 +1,136 @@
+from __future__ import annotations
+from dataclasses import dataclass, field
+
+
+@dataclass
+class Property:
+    name: str
+    data_type: str  # int32 | int64 | float32 | float64 | string | bool
+
+
+@dataclass
+class PropertyGroup:
+    properties: list[Property]
+    file_type: str = "parquet"
+    prefix: str = ""  # subdirectory name in chunk paths
+
+
+@dataclass
+class VertexInfo:
+    label: str
+    chunk_size: int
+    property_groups: list[PropertyGroup] = field(default_factory=list)
+    version: str = "gar/v1"
+
+    def to_dict(self) -> dict:
+        return {
+            "label": self.label,
+            "chunk_size": self.chunk_size,
+            "property_groups": [_pg_to_dict(pg) for pg in self.property_groups],
+            "version": self.version,
+        }
+
+    @classmethod
+    def from_dict(cls, d: dict) -> VertexInfo:
+        return cls(
+            label=d["label"],
+            chunk_size=d["chunk_size"],
+            property_groups=[_pg_from_dict(pg) for pg in d.get("property_groups", [])],
+            version=d.get("version", "gar/v1"),
+        )
+
+
+@dataclass
+class EdgeInfo:
+    src_type: str
+    edge_type: str
+    dst_type: str
+    chunk_size: int
+    src_chunk_size: int
+    directed: bool = True
+    property_groups: list[PropertyGroup] = field(default_factory=list)
+    version: str = "gar/v1"
+    # adj_lists stored so round-trips are lossless; v1 always emits ordered + unordered by src
+    adj_lists: list[dict] = field(default_factory=lambda: [
+        {"ordered": True,  "aligned_by": "src", "file_type": "parquet", "prefix": "ordered_by_source"},
+        {"ordered": False, "aligned_by": "src", "file_type": "parquet", "prefix": "unordered_by_source"},
+    ])
+
+    @property
+    def etype(self) -> tuple[str, str, str]:
+        return (self.src_type, self.edge_type, self.dst_type)
+
+    def to_dict(self) -> dict:
+        return {
+            "src_type": self.src_type,
+            "edge_type": self.edge_type,
+            "dst_type": self.dst_type,
+            "directed": self.directed,
+            "chunk_size": self.chunk_size,
+            "src_chunk_size": self.src_chunk_size,
+            "adj_lists": self.adj_lists,
+            "property_groups": [_pg_to_dict(pg) for pg in self.property_groups],
+            "version": self.version,
+        }
+
+    @classmethod
+    def from_dict(cls, d: dict) -> EdgeInfo:
+        return cls(
+            src_type=d["src_type"],
+            edge_type=d["edge_type"],
+            dst_type=d["dst_type"],
+            chunk_size=d["chunk_size"],
+            src_chunk_size=d["src_chunk_size"],
+            directed=d.get("directed", True),
+            property_groups=[_pg_from_dict(pg) for pg in d.get("property_groups", [])],
+            version=d.get("version", "gar/v1"),
+            adj_lists=d.get("adj_lists", [
+                {"ordered": True,  "aligned_by": "src", "file_type": "parquet", "prefix": "ordered_by_source"},
+                {"ordered": False, "aligned_by": "src", "file_type": "parquet", "prefix": "unordered_by_source"},
+            ]),
+        )
+
+
+@dataclass
+class GraphInfo:
+    name: str
+    prefix: str
+    vertex_infos: list[VertexInfo] = field(default_factory=list)
+    edge_infos: list[EdgeInfo] = field(default_factory=list)
+    version: str = "gar/v1"
+
+    def to_dict(self) -> dict:
+        return {
+            "name": self.name,
+            "prefix": self.prefix,
+            "vertices": [f"{vi.label}.vertex.yml" for vi in self.vertex_infos],
+            "edges": [f"{ei.src_type}_{ei.edge_type}_{ei.dst_type}.edge.yml" for ei in self.edge_infos],
+            "version": self.version,
+        }
+
+    @classmethod
+    def from_dict(cls, d: dict, vertex_infos: list[VertexInfo] | None = None, edge_infos: list[EdgeInfo] | None = None) -> GraphInfo:
+        """Reconstruct GraphInfo from manifest dict. Pass pre-loaded vertex/edge infos if available."""
+        return cls(
+            name=d["name"],
+            prefix=d.get("prefix", ""),
+            vertex_infos=vertex_infos or [],
+            edge_infos=edge_infos or [],
+            version=d.get("version", "gar/v1"),
+        )
+
+
+def _pg_to_dict(pg: PropertyGroup) -> dict:
+    return {
+        "properties": [{"name": p.name, "data_type": p.data_type} for p in pg.properties],
+        "file_type": pg.file_type,
+        "prefix": pg.prefix,
+    }
+
+
+def _pg_from_dict(d: dict) -> PropertyGroup:
+    return PropertyGroup(
+        properties=[Property(**p) for p in d["properties"]],
+        file_type=d.get("file_type", "parquet"),
+        prefix=d.get("prefix", ""),
+    )

deltagraphar/format/writer.py

@@ -0,0 +1,14 @@
+import io
+import yaml
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+
+def write_table(backend, path: str, table: pa.Table) -> None:
+    buf = io.BytesIO()
+    pq.write_table(table, buf)
+    backend.write_file(path, buf.getvalue())
+
+
+def write_yaml(backend, path: str, data: dict) -> None:
+    backend.write_file(path, yaml.dump(data, default_flow_style=False).encode())

deltagraphar/store/compaction.py

@@ -0,0 +1,119 @@
+from __future__ import annotations
+import pyarrow as pa
+import pyarrow.compute as pc
+
+from deltagraphar.format.paths import adj_list_chunk_path, offset_chunk_path, edge_prop_chunk_path
+from deltagraphar.format.reader import read_adj_list, count_rows, read_table
+from deltagraphar.format.writer import write_table
+
+
+def should_compact(backend, etype, vchunk, threshold_ratio, min_rows):
+    """True when delta rows exceed max(threshold_ratio * base_rows, min_rows)."""
+    src, et, dst = etype
+    delta_rows = count_rows(backend, src, et, dst, "unordered_by_source", vchunk)
+    if delta_rows == 0:
+        return False
+    base_rows = count_rows(backend, src, et, dst, "ordered_by_source", vchunk)
+    return delta_rows > max(threshold_ratio * base_rows, min_rows)
+
+
+def compact_vchunk(backend, etype, vchunk, vsize, edge_chunk_size, pg_prefixes):
+    """Merge unordered delta into ordered_by_source CSR and recompute offsets.
+
+    Only files under part<vchunk> change. Sort by (src_physical, dst_physical)
+    for deterministic layout.
+    """
+    src, et, dst = etype
+
+    base_adj = read_adj_list(backend, src, et, dst, "ordered_by_source", vchunk)
+    delta_adj = read_adj_list(backend, src, et, dst, "unordered_by_source", vchunk)
+
+    if len(base_adj) == 0 and len(delta_adj) == 0:
+        return
+
+    parts = [t for t in [base_adj, delta_adj] if len(t) > 0]
+    merged = pa.concat_tables(parts) if len(parts) > 1 else parts[0]
+
+    # Sort by (src, dst) — required for CSR correctness and determinism
+    sort_indices = pc.sort_indices(
+        merged, sort_keys=[("src_physical", "ascending"), ("dst_physical", "ascending")]
+    )
+    merged = merged.take(sort_indices)
+
+    # Write new ordered adj_list chunks
+    _write_chunks(
+        backend, merged, edge_chunk_size,
+        lambda ci: adj_list_chunk_path(src, et, dst, "ordered_by_source", vchunk, ci),
+    )
+
+    # Recompute CSR offset array: offsets[i] = index of first row with src_physical >= vstart+i
+    vstart = vchunk * vsize
+    src_arr = merged["src_physical"].to_pylist()
+    ptr = 0
+    offsets: list[int] = []
+    for i in range(vsize + 1):
+        target = vstart + i
+        while ptr < len(src_arr) and src_arr[ptr] < target:
+            ptr += 1
+        offsets.append(ptr)
+
+    write_table(
+        backend,
+        offset_chunk_path(src, et, dst, vchunk),
+        pa.table({"offset": pa.array(offsets, type=pa.int64())}),
+    )
+
+    # Compact edge property groups in tandem, preserving row alignment with adj_list
+    for pg_prefix in pg_prefixes:
+        base_pg = _read_all_pg_chunks(backend, src, et, dst, "ordered_by_source", pg_prefix, vchunk)
+        delta_pg = _read_all_pg_chunks(backend, src, et, dst, "unordered_by_source", pg_prefix, vchunk)
+        pg_parts = [t for t in [base_pg, delta_pg] if t is not None and len(t) > 0]
+        if pg_parts:
+            merged_pg = (pa.concat_tables(pg_parts) if len(pg_parts) > 1 else pg_parts[0]).take(sort_indices)
+            _write_chunks(
+                backend, merged_pg, edge_chunk_size,
+                lambda ci, _p=pg_prefix: edge_prop_chunk_path(src, et, dst, "ordered_by_source", _p, vchunk, ci),
+            )
+
+    # Truncate delta: overwrite all existing delta chunks with an empty table
+    _truncate_delta(backend, src, et, dst, vchunk, pg_prefixes)
+
+
+def _write_chunks(backend, table, chunk_size, path_fn):
+    n = len(table)
+    if n == 0:
+        write_table(backend, path_fn(0), table)
+        return
+    for ci, start in enumerate(range(0, n, chunk_size)):
+        write_table(backend, path_fn(ci), table.slice(start, chunk_size))
+
+
+def _read_all_pg_chunks(backend, src, et, dst, adj_type, pg_prefix, vchunk):
+    prefix = f"edge/{src}_{et}_{dst}/{adj_type}/{pg_prefix}/part{vchunk}"
+    paths = sorted(backend.list(prefix))
+    if not paths:
+        return None
+    return pa.concat_tables([read_table(backend, p) for p in paths])
+
+
+def _truncate_delta(backend, src, et, dst, vchunk, pg_prefixes=()):
+    """Zero out all unordered delta chunks for this vchunk (adj_list + all property groups)."""
+    empty_adj = pa.table({
+        "src_physical": pa.array([], type=pa.int64()),
+        "dst_physical": pa.array([], type=pa.int64()),
+    })
+    _zero_prefix(backend, f"edge/{src}_{et}_{dst}/unordered_by_source/adj_list/part{vchunk}", empty_adj)
+    for pg_prefix in pg_prefixes:
+        # empty table with no columns — schema unknown here, but zero rows is correct sentinel
+        _zero_prefix(backend, f"edge/{src}_{et}_{dst}/unordered_by_source/{pg_prefix}/part{vchunk}",
+                     pa.table({}))
+
+
+def _zero_prefix(backend, prefix, empty_table):
+    existing = backend.list(prefix)
+    if existing:
+        for p in existing:
+            write_table(backend, p, empty_table)
+    else:
+        # write a sentinel chunk0 so future compactions see "already cleared"
+        write_table(backend, prefix + "/chunk0", empty_table)