memry 0.1.0__py3-none-win_amd64.whl

memry/__init__.py

@@ -0,0 +1,13 @@
+"""
+Memry: Lightning-fast, in-memory, cross-process data frames for Python.
+Stop Pickling. Start Computing.
+"""
+__version__ = "0.1.0"
+
+# --- CORRECTED IMPORTS ---
+# We no longer import `start_server` from `memry.server` because that
+# function has been removed. The user now starts the server via the command line.
+# We also no longer need to import `shutdown_server` here, as it's not part of the primary API.
+
+from .client import put, get, delete, list_keys, close, shutdown_server
+from .exceptions import MemryError, ConnectionError, ServerError

memry/client.py

@@ -0,0 +1,148 @@
+import json
+import socket
+import io
+import pandas as pd
+import pyarrow as pa
+import pyarrow.ipc
+from .exceptions import ConnectionError, ServerError
+
+# --- CHANGED to a TCP Address ---
+SERVER_ADDRESS = ("127.0.0.1", 56789)
+BUFFER_SIZE = 8192
+
+class MemryConnection:
+    """Manages a persistent connection to the Memry daemon."""
+    def __init__(self):
+        self.sock = None
+        self.reader = None
+
+    def connect(self):
+        if self.sock and self.is_connected():
+            return
+        try:
+            # --- CHANGED to an INET (TCP) socket ---
+            self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            self.sock.connect(SERVER_ADDRESS)
+            self.reader = self.sock.makefile('rb')
+        except (socket.error, ConnectionRefusedError) as e:
+            self.sock = None
+            self.reader = None
+            raise ConnectionError(
+                f"Could not connect to Memry daemon at {SERVER_ADDRESS[0]}:{SERVER_ADDRESS[1]}. "
+                "Is the server running?"
+            ) from e
+            
+    def is_connected(self):
+        try:
+            # Create a temporary connection to check status without disrupting the main one
+            with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as test_sock:
+                test_sock.settimeout(0.5) # Don't wait forever
+                test_sock.connect(SERVER_ADDRESS)
+            return True
+        except (socket.error, ConnectionRefusedError):
+            return False
+
+    def close(self):
+        if self.sock:
+            try:
+                self.sock.close()
+            except socket.error:
+                pass
+            self.sock = None
+            self.reader = None
+
+    def _ensure_connection(self):
+        if not self.sock:
+            self.connect()
+        # Ping to check if the connection is still alive
+        try:
+            self._send_command_on_socket({'action': 'Ping'})
+            response = self._receive_response_from_reader()
+            if response.get('message') != 'pong':
+                raise ConnectionError("Connection lost")
+        except (socket.error, BrokenPipeError, ConnectionError):
+            # If ping fails, reconnect and try again
+            self.connect()
+
+    def _send_command_on_socket(self, cmd_dict):
+        payload = (json.dumps(cmd_dict) + '\n').encode('utf-8')
+        self.sock.sendall(payload)
+        
+    def _receive_response_from_reader(self):
+        line = self.reader.readline()
+        if not line:
+            raise ConnectionError("Daemon closed the connection unexpectedly.")
+        
+        resp = json.loads(line)
+        if resp.get('status') == 'Error':
+            raise ServerError(resp.get('message', 'Unknown server error'))
+        return resp
+
+    def _read_data(self, data_len):
+        return self.reader.read(data_len)
+
+# Global connection object for simple API
+_connection = MemryConnection()
+
+def put(df: pd.DataFrame) -> str:
+    _connection._ensure_connection()
+    if not isinstance(df, pd.DataFrame):
+        raise TypeError("Input must be a pandas DataFrame")
+
+    table = pa.Table.from_pandas(df, preserve_index=False)
+    sink = io.BytesIO()
+    with pa.ipc.new_stream(sink, table.schema) as writer:
+        writer.write_table(table)
+    
+    data_bytes = sink.getvalue()
+    
+    cmd = {"action": "Put", "data_len": len(data_bytes)}
+    _connection._send_command_on_socket(cmd)
+    _connection.sock.sendall(data_bytes)
+    
+    response = _connection._receive_response_from_reader()
+    return response['message']
+
+def get(key: str) -> pd.DataFrame:
+    _connection._ensure_connection()
+    cmd = {"action": "Get", "key": key}
+    _connection._send_command_on_socket(cmd)
+    
+    response = _connection._receive_response_from_reader()
+    data_len = response['data_len']
+    
+    data_bytes = _connection._read_data(data_len)
+
+    with pa.ipc.open_stream(data_bytes) as reader:
+        table = reader.read_all()
+    
+    return table.to_pandas()
+
+def delete(key: str):
+    _connection._ensure_connection()
+    cmd = {"action": "Delete", "key": key}
+    _connection._send_command_on_socket(cmd)
+    _connection._receive_response_from_reader()
+
+def list_keys() -> list:
+    _connection._ensure_connection()
+    cmd = {"action": "ListKeys"}
+    _connection._send_command_on_socket(cmd)
+    response = _connection._receive_response_from_reader()
+    return response.get('keys', [])
+
+def shutdown_server():
+    print("Sending shutdown command to Memry daemon...")
+    try:
+        _connection._ensure_connection()
+        cmd = {"action": "Shutdown"}
+        _connection._send_command_on_socket(cmd)
+        _connection._receive_response_from_reader()
+        print("Shutdown successful.")
+    except ConnectionError:
+        print("Could not connect to server. It might already be stopped.")
+    finally:
+        _connection.close()
+
+def close():
+    _connection.close()

memry/exceptions.py

@@ -0,0 +1,11 @@
+class MemryError(Exception):
+    """Base exception for the Memry client."""
+    pass
+
+class ConnectionError(MemryError):
+    """Raised when the client cannot connect to the Memry daemon."""
+    pass
+
+class ServerError(MemryError):
+    """Raised when the server returns an error message."""
+    pass

memry/server.py

@@ -0,0 +1,60 @@
+import subprocess
+import sys
+from pathlib import Path
+
+def get_server_binary_path() -> Path:
+    """
+    Finds the path to the packaged Rust binary ('memryd'), handling
+    both installed packages and local development environments. This function
+    is platform-aware and looks for '.exe' on Windows.
+    """
+    # Determine the correct binary name based on the operating system
+    if sys.platform == "win32":
+        binary_name = "memryd.exe"
+    else:
+        binary_name = "memryd"
+
+    # 1. Check the standard installation location first.
+    install_path = Path(__file__).parent / "bin" / binary_name
+    if install_path.exists():
+        return install_path
+
+    # 2. If not found, this is a development environment.
+    #    The binary is in the project's root `target` directory.
+    project_root = Path(__file__).parent.parent
+    dev_path = project_root / "target" / "debug" / binary_name
+    if dev_path.exists():
+        return dev_path
+
+    # Check release as a final fallback
+    dev_path_release = project_root / "target" / "release" / binary_name
+    if dev_path_release.exists():
+        return dev_path_release
+    
+    # 3. If it's in none of those places, the build is broken.
+    raise FileNotFoundError(
+        f"Memry server binary ('{binary_name}') could not be found in the installation "
+        "or development target directories. Please run 'maturin develop'."
+    )
+
+def main():
+    """
+    Finds and runs the compiled Memry daemon, letting it take over the
+    current terminal. Press Ctrl+C in the terminal to stop the server.
+    """
+    try:
+        server_path = get_server_binary_path()
+        print(f"--- Launching Memry Server from: {server_path} ---")
+        subprocess.run([server_path], check=True)
+    except FileNotFoundError as e:
+        print(f"ERROR: {e}", file=sys.stderr)
+        sys.exit(1)
+    except subprocess.CalledProcessError as e:
+        print(f"ERROR: Memry server exited with an error (code {e.returncode}).", file=sys.stderr)
+        sys.exit(1)
+    except KeyboardInterrupt:
+        print("\nLauncher interrupted. Server shut down.")
+        sys.exit(0)
+
+if __name__ == "__main__":
+    main()

memry-0.1.0.dist-info/METADATA

@@ -0,0 +1,155 @@
+Metadata-Version: 2.4
+Name: memry
+Version: 0.1.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Rust
+Requires-Dist: pandas>=1.0
+Requires-Dist: pyarrow>=10.0
+License-File: LICENSE
+Summary: Lightning-fast, in-memory, cross-process data store for Python
+Author-email: Your Name <your@email.com>
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Homepage, https://github.com/your-username/memry
+Project-URL: Repository, https://github.com/your-username/memry
+
+# Project Memry
+
+![Python Version](https://img.shields.io/pypi/pyversions/memry)
+![PyPI Version](https://img.shields.io/pypi/v/memry.svg)
+![Build Status](https://img.shields.io/github/actions/workflow/status/your-username/memry/ci.yml?branch=main)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+### *Stop Pickling. Start Computing.*
+
+**Memry** is a lightning-fast, in-memory, cross-process data store for Python, designed to eliminate the crippling overhead of serialization (`pickle`) in multiprocessing workflows.
+
+## The Problem
+
+When using Python's `multiprocessing` library, sharing large objects like Pandas DataFrames between processes is incredibly slow. Python must `pickle` the data in the main process, send the bytes over a pipe, and `unpickle` it in the child process. For gigabytes of data, this overhead can make your parallel code slower than single-threaded code.
+
+## The Solution: Memry
+
+Memry runs a tiny, high-performance daemon (written in Rust) that manages a block of memory. Your Python processes can `put` and `get` data from this central store.
+
+Instead of sending your huge DataFrame to each process, you store it in Memry **once** and then pass a tiny string key to your child processes. Data transfer is nearly instantaneous because it's handled by a highly optimized system using the Apache Arrow format.
+
+**Analogy:** Instead of photocopying a 1,000-page book for every colleague (pickling), you place the book on a shared library shelf and just tell them its location (Memry).
+
+
+*(You would create and upload a simple diagram showing this flow)*
+
+## Benchmarks
+
+The results speak for themselves. This benchmark measures the time taken to send a Pandas DataFrame to 4 worker processes and have them access it.
+
+| DataFrame Size | `multiprocessing` (Pickle) | **`memry`** (Arrow IPC) | Speedup |
+| :------------- | :------------------------- | :---------------------- | :------ |
+| 10 MB          | 0.15 s                     | **0.008 s**             | **18x** |
+| 100 MB         | 1.48 s                     | **0.07 s**              | **21x** |
+| 500 MB         | 7.9 s                      | **0.35 s**              | **22x** |
+| 1 GB           | 17.2 s                     | **0.71 s**              | **24x** |
+
+*(These are realistic estimates. Run `examples/benchmark.py` to generate your own.)*
+
+<img src="https://i.imgur.com/your-benchmark-graph.png" alt="Benchmark Graph" width="600"/>
+*(You would generate and upload a graph plotting this data)*
+
+## Installation
+
+```bash
+pip install memry
+```
+
+## Quick Start
+
+**1. Start the Memry Server**
+
+From your terminal:
+```bash
+memry-server start
+```
+*To stop the server, run `memry-server stop`.*
+
+**2. Use it in your Python code**
+
+Compare the standard, slow way with the revolutionary Memry way.
+
+**Before Memry (The Pain):**
+```python
+import multiprocessing as mp
+import pandas as pd
+import numpy as np
+import time
+
+def process_chunk(df):
+    # This is slow because 'df' had to be pickled and sent.
+    return df['value'].sum()
+
+if __name__ == "__main__":
+    large_df = pd.DataFrame(np.random.rand(10_000_000, 1), columns=['value'])
+
+    start_time = time.time()
+    with mp.Pool(processes=4) as pool:
+        # This is the slow part! Pickling and transferring the data.
+        results = pool.map(process_chunk, [large_df] * 4)
+    print(f"Standard multiprocessing took: {time.time() - start_time:.2f}s")
+```
+
+**After Memry (The Revolution):**
+```python
+import multiprocessing as mp
+import pandas as pd
+import numpy as np
+import time
+import memry
+
+def process_chunk_from_memry(data_key):
+    # Getting data is INSTANTANEOUS (no unpickling!)
+    df = memry.get(data_key)
+    return df['value'].sum()
+
+if __name__ == "__main__":
+    large_df = pd.DataFrame(np.random.rand(10_000_000, 1), columns=['value'])
+    
+    start_time = time.time()
+    # 1. Put the data into Memry ONCE. This is fast.
+    data_key = memry.put(large_df)
+
+    # 2. Pass only the tiny key string.
+    with mp.Pool(processes=4) as pool:
+        results = pool.map(process_chunk_from_memry, [data_key] * 4)
+    
+    print(f"Memry multiprocessing took: {time.time() - start_time:.2f}s")
+    
+    # 3. Clean up the data from Memry.
+    memry.delete(data_key)
+```
+
+## API
+
+- `memry.put(df: pd.DataFrame) -> str`: Store a DataFrame, get a key.
+- `memry.get(key: str) -> pd.DataFrame`: Retrieve a DataFrame using its key.
+- `memry.delete(key: str)`: Remove a DataFrame from the store.
+- `memry.list_keys() -> list`: List all keys in the store.
+- `memry.close()`: Close the client's connection to the daemon.
+- `memry.start_server()`: Start the server from within Python.
+- `memry.shutdown_server()`: Stop the server from within Python.
+
+## Roadmap
+
+This is just the beginning. The vision for Memry includes:
+-   [ ] **True Zero-Copy:** Transition from Unix sockets to shared memory for even faster reads.
+-   [ ] **In-Memory Querying:** Run SQL-like queries directly on the stored Arrow data without moving it into Python.
+-   [ ] **Cross-Language Support:** Clients for Julia, R, etc.
+-   [ ] **Persistence:** Snapshotting the in-memory database to disk.
+
+## Contributing
+
+Contributions are welcome! Please open an issue or submit a pull request.
+
+## License
+
+This project is licensed under the MIT License.

memry-0.1.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+memry-0.1.0.data/scripts/memryd.exe,sha256=NV0kWUlan0BouZv-qHWjDyz8uA6QEKfIx5jUpWx9fmQ,2020352
+memry-0.1.0.dist-info/METADATA,sha256=sKolpV1bNB5e0ksOcj3X9DM0EZyb4gdVj1qqP4IaY9U,6101
+memry-0.1.0.dist-info/WHEEL,sha256=otkNWQo5-3v8ZzaSqBicFgqaRvPB-IMXst8ik5MzIfo,93
+memry-0.1.0.dist-info/licenses/LICENSE,sha256=q53YqEH5OACuJ8YmE3i9pND509hapVaOX42ix2AMkZ8,1085
+memry/__init__.py,sha256=o9nq3BcusK3gB3Aq-eHYWP-51lKqpZZ091kZOBt1EKE,560
+memry/__pycache__/__init__.cpython-311.pyc,sha256=Rht705OZKls0BIbevQwzCuK3s_VYE6Om2Ubragx69ss,619
+memry/__pycache__/client.cpython-311.pyc,sha256=jP0rl63Z2U6uGAdMnD9MH-pRDMrYAs4yf60nAzhZUG0,9760
+memry/__pycache__/exceptions.cpython-311.pyc,sha256=OtyiKgGthrpKEmHy46anKDz-96z___7zx_FFECLGDag,981
+memry/__pycache__/server.cpython-311.pyc,sha256=kJ84drEKdmiYFO3kmSKGZ7s0zEL2vs-Fifo7imeKFbg,3029
+memry/client.py,sha256=8p6YukOYHAq-EnJ4uuRFcQ-4QKnl3z6XsHX1m1PDjew,5035
+memry/exceptions.py,sha256=MwEOP3RqnrkQiq-COmQqu_RTKB7VP9JfkjBP7TlhCGI,308
+memry/server.py,sha256=AgS0WJParqrkSU4-XIlJLHYJ2nJVOe1xvf63un1NUw0,2218
+memry-0.1.0.dist-info/RECORD,,

memry-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: maturin (1.8.7)
+Root-Is-Purelib: false
+Tag: py3-none-win_amd64

memry-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Your Name
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.