graphzero 0.2.0__cp313-cp313-win_amd64.whl

graphzero/__init__.py

@@ -0,0 +1,15 @@
+from .graphzero import Graph, convert_csv_to_gl, convert_csv_to_gd, DataType, FeatureStore
+
+# Metadata
+__version__ = "0.2.0"
+__author__ = "Krish Singaria"
+__license__ = "MIT"
+
+# Define what happens when someone does "from graphzero import *"
+__all__ = [
+    "Graph",
+    "convert_csv_to_gl",
+    "convert_csv_to_gd",
+    "DataType",
+    "FeatureStore"
+]

graphzero/graphzero.pyi

@@ -0,0 +1,170 @@
+import enum
+from typing import List, Tuple, Any, Union
+import numpy as np
+import numpy.typing as npt
+
+class Graph:
+    """
+    Graph Class contains the graphfile and its relevant functions and methods.
+    It holds the mmap / zero-copy memory.
+    """
+    
+    num_nodes: int
+    num_edges: int
+
+    def __init__(self, filename: str) -> None:
+        """
+        Args:
+            filename (str): either absolute path or relative path (depends on the current working directory).
+        Returns:
+            Graph class instance.
+        """
+        ...
+
+    def get_degree(self, node_id: int) -> int:
+        """
+        Get the degree of a node.
+        
+        Args:
+            node_id (int)
+        Returns:
+            degree (int)
+        """
+        ...
+
+    def get_neighbours(self, node_id: int) -> npt.NDArray[np.uint64]:
+        """
+        Returns the neighbours of a node.
+        
+        Args:
+            node_id (int)
+        Returns:
+            1-D numpy ndarray of neighbour node IDs (dtype: platform-size integer).
+        """
+        ...
+
+    def batch_random_walk(self, start_nodes: List[int], walk_length: int, p: float = 1.0, q: float = 1.0) -> npt.NDArray[np.int64]:
+        """
+        Performs 2nd-order random walks (Node2Vec style).
+
+        Args:
+            start_nodes (list): list of starting nodes.
+            walk_length (int): how long walks should be (e.g. 10).
+            p (float): Return parameter; Low = keeps walk local (BFS-like).
+            q (float): In-out parameter; Low = explores far away (DFS-like).
+
+        Returns:
+            ndarray of shape (len(start_nodes), walk_length) with dtype np.int64.
+        """
+        ...
+
+    def batch_random_walk_uniform(self, start_nodes: List[int], walk_length: int) -> npt.NDArray[np.int64]:
+        """
+        Performs uniform random walks.
+
+        Args:
+            start_nodes (list): list of starting nodes.
+            walk_length (int): how long walks should be (e.g. 10).
+
+        Returns:
+            ndarray of shape (len(start_nodes), walk_length) with dtype np.int64.
+        """
+        ...
+
+    def batch_random_fanout(self, start_nodes: List[int], K: int) -> npt.NDArray[np.int64]:
+        """
+        Performs uniform random fanout sampling.
+
+        Args:
+            start_nodes (list): list of starting nodes.
+            K (int): how many neighbours to sample.
+
+        Returns:
+            ndarray of shape (len(start_nodes), K) with dtype np.int64.
+        """
+        ...
+
+    def sample_neighbours(self, start_node: int, K: int) -> npt.NDArray[np.int64]:
+        """
+        Performs uniform random neighbour sampling for a node.
+
+        Args:
+            start_node (int): node id to sample from.
+            K (int): how many neighbours to sample.
+
+        Returns:
+            1-D ndarray with up to K neighbour ids (dtype np.int64).
+        """
+        ...
+
+    def __getstate__(self) -> tuple: ...
+    def __setstate__(self, state: tuple) -> None: ...
+
+
+class DataType(enum.Enum):
+    """
+    Enum DataType only For FeatureStore datatype
+    
+    use gz.DataType.INT32, gz.DataType.INT64, gz.DataType.FLOAT32, gz.DataType.FLOAT64 when converting csv to gd.
+    """
+    INT32 = 0,
+    INT64 = 1,
+    FLOAT32 = 2,
+    FLOAT64 = 3
+    
+
+class FeatureStore:
+    """
+    FeatureStore Class contains the datafile and its relevant functions and methods.
+    It holds the mmap / zero-copy memory to Features Matrix.
+    """
+    num_nodes: int
+    feature_dim: int
+    
+    def __init__(self, filename: str) -> None:
+        """
+        Args:
+            filename (str): either absolute path or relative path (depends on the current working directory).
+        Returns:
+            FeatureStore class instance.
+        """
+        ...
+    def get_data(self, node_id: int) -> npt.NDArray[Any]:
+        """
+        Get the features of a node ID.
+
+        Args:
+            node_id (int): Node ID to retrieve features for.
+        Returns:
+            1-D numpy ndarray of shape (feature_dim,).
+        """
+        ...
+    def get_tensor(self) -> npt.NDArray[Any]:
+        """
+        Returns the entire Data (tensor).
+        Returns:
+            2-D numpy ndarray of shape (num_nodes, feature_dim) with the given dtype.
+        """
+        ...
+
+def convert_csv_to_gl(csv_path: str, out_path: str, directed: bool = False) -> None:
+    """
+    Convert a CSV edge list to GraphZero binary format (.gl)
+
+    Args:
+        csv_path (str): Path to input CSV.
+        out_path (str): Path to output .gl file.
+        directed (bool): Whether the graph is directed. Defaults to False.
+    """
+    ...
+    
+def convert_csv_to_gd(csv_path: str, out_path: str, dtype: DataType) -> None:
+    """
+    Convert a CSV data to GraphZero data format (.gd).
+
+    Args:
+        csv_path (str): Path to input CSV.
+        out_path (str): Path to output .gd file.
+        dtype (DataType): Data type of the features MUST BE SET BY THE USER.
+    """
+    ...

graphzero-0.2.0.dist-info/METADATA

@@ -0,0 +1,167 @@
+Metadata-Version: 2.2
+Name: graphzero
+Version: 0.2.0
+Summary: High-performance Zero-Copy Graph Engine
+Author-Email: Krish <krishsingaria2005@gmail.com>
+Requires-Python: >=3.8
+Requires-Dist: numpy
+Requires-Dist: torch
+Description-Content-Type: text/markdown
+
+# GraphZero
+
+**High-Performance, Zero-Copy Graph Engine for Massive Datasets on Consumer Hardware.**
+
+GraphZero is a C++ graph processing engine with lightweight Python bindings designed to solve the **"Memory Wall"** in Graph Neural Networks (GNNs). It allows you to load and sample **100 Million+ node graphs** (like `ogbn-papers100M`) and their massive feature matrices on a standard 16GB RAM laptop—something standard libraries like PyTorch Geometric (PyG) or DGL cannot do.
+
+## The Problem
+
+GNN datasets can be massive. `ogbn-papers100M` contains **111 Million nodes**, **1.6 Billion edges**, and gigabytes of node embeddings.
+
+* **Standard approach (PyG/NetworkX):** Tries to load the entire graph structure and all node features into **RAM** before training begins.
+* **The Result:** `MemoryError` (OOM) on consumer hardware. You need 64GB+ **RAM** servers just to *load* the data.
+
+## The Solution:
+
+GraphZero abandons the "Load-to-RAM" model. Instead, it uses a custom **Zero-Copy Architecture**:
+
+* **Memory Mapping (`mmap`):** The graph and its features stay on disk. The OS only loads the specific "hot" pages needed for computation into RAM via page faults.
+* **Compressed CSR (`.gl`):** A custom binary format that compresses raw edges by **~60%** (30GB CSV $\to$ 13GB Binary).
+* **Columnar Tensor Store (`.gd`):** A raw, C-contiguous binary format for node features that instantly translates to PyTorch tensors without memory allocation.
+* **Parallel Sampling:** OpenMP-accelerated random walks that saturate NVMe SSD throughput, using thread-local RNGs to eliminate lock contention.
+
+## 🏆 Benchmarks: GraphZero vs. PyTorch Geometric
+
+**Task:** Load `ogbn-papers100M` (56GB Raw) and perform random walks.
+**Hardware:** Windows Laptop (16GB RAM, NVMe SSD).
+
+| Metric | GraphZero (v0.2) | PyTorch Geometric |
+| --- | --- | --- |
+| **Load Time** | **0.000000 s** ⚡ | **FAILED** (Crash) ❌ |
+| **Peak RAM Usage** | **~5.1 GB** (OS Cache) | **>24.1 GB** (Required) |
+| **Throughput** | **1,264,000 steps/s** | N/A |
+| **Status** | ✅ **Success** | ❌ **OOM Error** |
+
+### Proof of Performance
+
+> *Left: GraphZero loading instantly and utilizing OS Page Cache. Right: PyG crashing with `Unable to allocate 24.1 GiB`.*
+
+<p float="left ">
+<img src="benchmark/images/gz_bench.png" width="45%" />
+<img src="benchmark/images/py_crash.png" width="45%" />
+</p>
+
+
+## 📦 Installation
+
+GraphZero is available on PyPI:
+
+```bash
+pip install graphzero
+
+```
+
+
+## 🚀 Quick Start
+
+### 1. Convert Your Data (Topology & Features)
+
+GraphZero uses high-efficiency binary formats. Convert your generic CSV lists once.
+
+example `edges.csv`, weights are optional:
+```csv
+src,dst,weight
+0,1,0.5
+1,2,1.0
+```
+
+```python
+import graphzero as gz
+
+# 1. Convert Topology (Edges & Weights) to .gl
+gz.convert_csv_to_gl(
+    input_csv="dataset/edges.csv", 
+    output_bin="graph.gl", 
+    directed=True
+)
+
+# 2. Convert Node Features to .gd (Float32, Int64, etc.)
+gz.convert_csv_to_gd(
+    csv_path="dataset/features.csv",
+    out_path="features.gd",
+    dtype=gz.DataType.FLOAT32
+)
+
+```
+
+### 2. High-Speed Sampling & Zero-Copy Tensors
+
+Once converted, the graph and its multi-gigabyte feature matrix are instantly accessible without consuming RAM.
+
+```python
+import graphzero as gz
+import numpy as np
+
+# TOPOLOGY
+# Zero-Copy Load (Instant)
+g = gz.Graph("graph.gl")
+
+# Define Start Nodes
+start_nodes = np.random.randint(0, g.num_nodes, 1000).astype(np.uint64)
+
+# Parallel Biased Random Walk (Node2Vec style: p=1.0, q=0.5)
+walks = g.batch_random_walk(
+    start_nodes=start_nodes, 
+    walk_length=10,
+    p=1.0, 
+    q=0.5
+)
+
+# FEATURES
+# Zero-Copy Feature Load (Instant)
+fs = gz.FeatureStore("features.gd")
+
+# Get a perfect 2D Numpy/PyTorch Tensor mapping directly to the SSD
+# RAM used: 0 Bytes!
+node_features = fs.get_tensor() 
+
+print(f"Graph loaded. Feature Matrix Shape: {node_features.shape}")
+
+```
+
+## ⚙️ Under the Hood
+
+GraphZero is built for **Systems & GNN** enthusiasts.
+
+* **Core:** C++20 with `nanobind` for Python bindings.
+* **Parallelism:** Uses `#pragma omp` with thread-local deterministic RNGs.
+* **IO:** Direct `CreateFileMapping` (Windows) and `mmap` (Linux) calls with alignment optimization (4KB/2MB pages).
+
+## 🌟 Current Features List (v0.2)
+
+GraphZero currently supports the following high-performance ML capabilities:
+
+**Graph Structural Engine**
+
+* **Instant Ingestion:** Fast `mmap`-backed loading of directed, undirected, and weighted graphs.
+* **Zero-Copy CSR:** Custom `.gl` binary format for dense, continuous memory alignment and 64-byte CPU cache line optimization.
+* **Thread-Safe Sampling:** OpenMP-accelerated `batch_random_walk_uniform` and `batch_random_fanout`.
+* **Biased Walks (Node2Vec):** Hardware-optimized Alias Table generation for $O(1)$ weighted sampling (`batch_random_walk` with `p` and `q` parameters).
+* **Fault-Tolerant:** Automatic handling of dead-ends (sinks) and out-of-bounds nodes.
+
+**Graph Data Engine**
+
+* **Columnar Tensor Store:** Custom `.gd` binary format for storing $N \times F$ feature matrices.
+* **Strong Typing:** Native C++ template dispatching supporting `FLOAT32`, `FLOAT64`, `INT32`, and `INT64`.
+* **Zero-Copy Bridge:** Direct translation of `mmap` pointers to Numpy/PyTorch multidimensional arrays.
+
+# 🗺️ Roadmap
+- v0.3 (The Algorithmic Core): High-performance analytics engine adding OpenMP-accelerated Parallel BFS/DFS, PageRank, and Connected Components.
+
+- v0.4 (Dynamic Updates): Breaking the immutable CSR barrier via an LSM-Tree/Adjacency List memory overlay to allow real-time edge/node insertions.
+
+- v0.5 (Production Hardening): ACID-compliant safety for multi-process PyTorch training using Reader-Writer Locks, Write-Ahead Logging (WAL), and graceful exception handling.
+
+## 📄 License
+
+MIT License. Created by **Krish Singaria** (IIT Mandi).

graphzero-0.2.0.dist-info/RECORD

@@ -0,0 +1,7 @@
+graphzero/__init__.py,sha256=lQhujCJEdJ5NlEZpfr2nkqmP-SFMpFF1jVX7U8sgEkU,367
+graphzero/graphzero.cp313-win_amd64.pyd,sha256=IaBWdT3PLYVFWu4rKHNyLSGJJOX37YdYwc7345qN9rg,206336
+graphzero/graphzero.pyi,sha256=wp3QbBetsxvnQNqblkTrKAfK9N4DvWqIRBfgGDz-hvE,5121
+graphzero-0.2.0.dist-info/METADATA,sha256=t-QpVocD59fxNUirxVSyICZuApHinypF4boQFU55wco,6092
+graphzero-0.2.0.dist-info/WHEEL,sha256=UZrbbE4r80xj7Ncfa6JoeTVe-77bdXLkKUA63V8pKWQ,106
+graphzero-0.2.0.dist-info/licenses/LICENSE,sha256=3f1lwtCZQI2fiQt-KWJ98nFb5EGrjZQzz-wZ62p-T8w,1081
+graphzero-0.2.0.dist-info/RECORD,,

graphzero-0.2.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: scikit-build-core 0.12.2
+Root-Is-Purelib: false
+Tag: cp313-cp313-win_amd64
+

graphzero-0.2.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Krish
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.