ds-service-client 0.2.0__tar.gz

ds_service_client-0.2.0/.clang-format

@@ -0,0 +1,10 @@
+BasedOnStyle: LLVM
+IndentWidth: 4
+ColumnLimit: 120
+DerivePointerAlignment: false
+PointerAlignment: Left
+BreakConstructorInitializers: BeforeComma
+SortIncludes: false
+AllowShortFunctionsOnASingleLine: Empty
+BreakTemplateDeclarations: Yes
+AlignEscapedNewlines: Left

ds_service_client-0.2.0/.cpush.json5

@@ -0,0 +1,6 @@
+{
+    project: "ds-service",
+    remotes: {
+        rivanna: "rivanna:ds-service"
+    }
+}

ds_service_client-0.2.0/.gitignore

@@ -0,0 +1,284 @@
+# Prerequisites
+*.d
+
+# Compiled Object files
+*.slo
+*.lo
+*.o
+*.obj
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Linker files
+*.ilk
+
+# Debugger Files
+*.pdb
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+*.so.*
+
+
+# Fortran module files
+*.mod
+*.smod
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*.lib
+
+# Executables
+*.exe
+*.out
+*.app
+
+# Build directories
+build/
+Build/
+build-*/
+
+# CMake generated files
+CMakeFiles/
+CMakeCache.txt
+cmake_install.cmake
+Makefile
+install_manifest.txt 
+compile_commands.json
+
+# Temporary files
+*.tmp 
+*.log 
+*.bak 
+*.swp 
+
+# vcpkg
+vcpkg_installed/
+
+# debug information files
+*.dwo
+
+# test output & cache
+Testing/
+.cache/# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#   Usually these files are written by a python script from a template
+#   before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+# Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+# uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+# poetry.lock
+# poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+# pdm.lock
+# pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+# pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# Redis
+*.rdb
+*.aof
+*.pid
+
+# RabbitMQ
+mnesia/
+rabbitmq/
+rabbitmq-data/
+
+# ActiveMQ
+activemq-data/
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#   JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#   be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#   and can be added to the global gitignore or merged into this file.  For a more nuclear
+#   option (not recommended) you can uncomment the following to ignore the entire idea folder.
+# .idea/
+
+# Abstra
+#   Abstra is an AI-powered process automation framework.
+#   Ignore directories containing user credentials, local state, and settings.
+#   Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#   Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#   that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#   and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#   you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+
+# Streamlit
+.streamlit/secrets.toml

ds_service_client-0.2.0/CMakeLists.txt

@@ -0,0 +1,58 @@
+cmake_minimum_required(VERSION 4.0)
+
+project(
+  ds-service
+  VERSION 0.0.1
+  LANGUAGES CXX)
+
+find_package(OpenMP REQUIRED)
+
+find_package(spdlog REQUIRED)
+find_package(argparse REQUIRED)
+find_package(phmap REQUIRED)
+find_package(gRPC REQUIRED)
+find_package(protobuf REQUIRED)
+
+find_program(PROTOC_EXE protoc REQUIRED)
+find_program(GRPC_CPP_PLUGIN_EXE grpc_cpp_plugin REQUIRED)
+
+add_custom_command(
+  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ds-service.pb.h
+         ${CMAKE_CURRENT_BINARY_DIR}/ds-service.pb.cc
+         ${CMAKE_CURRENT_BINARY_DIR}/ds-service.grpc.pb.h
+         ${CMAKE_CURRENT_BINARY_DIR}/ds-service.grpc.pb.cc
+  COMMAND
+    ${PROTOC_EXE} --cpp_out=${CMAKE_CURRENT_BINARY_DIR}
+    --grpc_out=${CMAKE_CURRENT_BINARY_DIR}
+    --plugin=protoc-gen-grpc=${GRPC_CPP_PLUGIN_EXE}
+    --proto_path=${CMAKE_CURRENT_SOURCE_DIR}/misc
+    ${CMAKE_CURRENT_SOURCE_DIR}/misc/ds-service.proto
+  DEPENDS misc/ds-service.proto
+  VERBATIM)
+
+add_library(ds-service-grpc ${CMAKE_CURRENT_BINARY_DIR}/ds-service.pb.cc
+                            ${CMAKE_CURRENT_BINARY_DIR}/ds-service.grpc.pb.cc)
+target_sources(
+  ds-service-grpc
+  PUBLIC FILE_SET
+         HEADERS
+         BASE_DIRS
+         ${CMAKE_CURRENT_BINARY_DIR}
+         FILES
+         ${CMAKE_CURRENT_BINARY_DIR}/ds-service.pb.h
+         ${CMAKE_CURRENT_BINARY_DIR}/ds-service.grpc.pb.h)
+target_link_libraries(ds-service-grpc PUBLIC protobuf::protobuf grpc::grpc)
+target_compile_features(ds-service-grpc PRIVATE cxx_std_17)
+
+add_executable(ds-server cpp/ds-server.cpp)
+target_sources(ds-server PRIVATE FILE_SET HEADERS BASE_DIRS cpp)
+target_link_libraries(
+  ds-server
+  PRIVATE spdlog::spdlog
+          argparse::argparse
+          phmap
+          ds-service-grpc
+          OpenMP::OpenMP_CXX)
+target_compile_features(ds-server PRIVATE cxx_std_23)
+
+install(TARGETS ds-server)

ds_service_client-0.2.0/LICENSE

@@ -0,0 +1,19 @@
+Copyright (C) 2026 Rector and Visitors of the University of Virginia
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ds_service_client-0.2.0/PKG-INFO

@@ -0,0 +1,144 @@
+Metadata-Version: 2.4
+Name: ds-service-client
+Version: 0.2.0
+Summary: Python client for DS service.
+Author-email: Parantapa Bhattacharya <pb@parantapa.net>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/parantapa/ds-service
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: grpcio>=1.60
+Requires-Dist: protobuf>=6.31
+Dynamic: license-file
+
+# ds-service: Yet Another Data Structure Server
+
+`ds-service` is a small, in-memory data structure server that is accessible via [gRPC](https://grpc.io/).
+
+`ds-service` runs a single server process (`ds-server`)
+that holds shared state in memory
+and lets many distributed clients and workers coordinate using it.
+
+Presently, it provides two things:
+- **A key-value store** -- a shared `string -> bytes` store
+    for passing data between processes.
+- **A task queue** -- a priority-based work queue
+    that distributes tasks to workers,
+    tracks their state,
+    and requeues tasks whose workers have died or stalled.
+
+It is designed as a lightweight coordination system for distributed batch jobs
+(for example, when running calibration and projection workflows across many nodes of an HPC cluster),
+where you need some shared data structures
+and a way to hand work out to a pool of workers.
+
+## Architecture
+
+- **Server** (`cpp/ds-server.cpp`) -- a C++23 gRPC service.
+    All state lives in memory and is guarded by a single global lock,
+    so operations are serialized and consistent.
+    State is **not** persisted; that is, when the server stops all data is lost.
+- **Client** (`python/ds_service_client/`) -- a Python 3.12+ client library
+    that wraps the generated gRPC stubs
+    and translates gRPC status codes into ordinary Python exceptions
+    (`KeyError`, `ValueError`, `TimeoutError`).
+- **Interface** (`misc/ds-service.proto`) -- the protobuf/gRPC contract
+    shared by both sides.
+
+By default the server listens on `127.0.0.1:5051`.
+
+## The key-value store
+
+A flat `string -> bytes` key-value store.
+
+| RPC | Description |
+| --- | --- |
+| `MapSet(key, value)` | Store `value` under `key`, overwriting any existing value. |
+| `MapGet(key)` | Return the value for `key`, or `NOT_FOUND` if it is missing. |
+
+Values are opaque bytes, so callers are free to store whatever serialization
+they like (JSON, pickle, protobuf, raw binary).
+
+## The task queue
+
+Tasks are units of work identified by a unique `task_id`.
+Each task carries an opaque `function` and `input` payload,
+a floating-point `priority`,
+and one or more named queues it should be dispatched from.
+A task moves through three states: `Ready` → `Running` → `Complete`.
+
+| RPC | Description |
+| --- | --- |
+| `TaskAdd(task_id, queue, priority, function, input)` | Register a new task and enqueue it on each named queue. Returns `ALREADY_EXISTS` if the id is already known. |
+| `TaskGet(worker_id, queue)` | Claim the highest-priority `Ready` task from the first non-empty queue, mark it `Running`, and return its payload. Returns `UNAVAILABLE` when no work is ready. |
+| `TaskDone(task_id, output)` | Mark a `Running` task `Complete` and store its output. |
+| `TaskStatus(task_id)` | Return a task's current state and, once complete, its output. |
+| `Requeue(timeout_s)` | Reset any task that has been `Running` longer than `timeout_s` back to `Ready` and re-enqueue it. |
+
+Within a queue, higher `priority` values are dispatched first.
+A worker polls `TaskGet` across the queues it cares about, runs the work,
+and reports back with `TaskDone`.
+`Requeue` provides fault tolerance:
+if a worker crashes without completing its task,
+a periodic `Requeue` call makes that task available to another worker.
+
+## Building the server
+
+Dependencies are managed with [Conan](https://conan.io/)
+and the build is driven by CMake.
+
+```sh
+conan install .
+. build/Release/generators/conanbuild.sh
+cmake -S . -B build/Release \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_TOOLCHAIN_FILE=generators/conan_toolchain.cmake
+cmake --build build/Release --parallel
+```
+
+A reproducible container build is defined in
+`scripts/apptainer/ds-server.def`.
+
+### Running
+
+```sh
+ds-server --address 0.0.0.0:5051
+```
+
+Run `ds-server --help` for the full argument list.
+
+## Python client
+
+```sh
+pip install .  # installs the ds-service-client package
+```
+
+```python
+from ds_service_client import Client, TaskState
+
+client = Client("127.0.0.1:5051")  # or set DS_SERVER_ADDRESS and call Client()
+
+# Key-value map
+client.map_set("greeting", b"hello")
+assert client.map_get("greeting") == b"hello"
+
+# Task queue
+client.task_add("job-1", queue="work", priority=1.0, function=b"...", input=b"...")
+
+task = client.task_get(worker_id="worker-a", queue="work")
+# ... do the work ...
+client.task_done(task.task_id, output=b"result")
+
+status = client.task_status("job-1")
+assert status.state == TaskState.Complete
+```
+
+If `Client()` is constructed without an address, it reads the server address
+from the `DS_SERVER_ADDRESS` environment variable.
+
+## License
+
+MIT -- see [LICENSE](LICENSE).

ds_service_client-0.2.0/README.md

@@ -0,0 +1,128 @@
+# ds-service: Yet Another Data Structure Server
+
+`ds-service` is a small, in-memory data structure server that is accessible via [gRPC](https://grpc.io/).
+
+`ds-service` runs a single server process (`ds-server`)
+that holds shared state in memory
+and lets many distributed clients and workers coordinate using it.
+
+Presently, it provides two things:
+- **A key-value store** -- a shared `string -> bytes` store
+    for passing data between processes.
+- **A task queue** -- a priority-based work queue
+    that distributes tasks to workers,
+    tracks their state,
+    and requeues tasks whose workers have died or stalled.
+
+It is designed as a lightweight coordination system for distributed batch jobs
+(for example, when running calibration and projection workflows across many nodes of an HPC cluster),
+where you need some shared data structures
+and a way to hand work out to a pool of workers.
+
+## Architecture
+
+- **Server** (`cpp/ds-server.cpp`) -- a C++23 gRPC service.
+    All state lives in memory and is guarded by a single global lock,
+    so operations are serialized and consistent.
+    State is **not** persisted; that is, when the server stops all data is lost.
+- **Client** (`python/ds_service_client/`) -- a Python 3.12+ client library
+    that wraps the generated gRPC stubs
+    and translates gRPC status codes into ordinary Python exceptions
+    (`KeyError`, `ValueError`, `TimeoutError`).
+- **Interface** (`misc/ds-service.proto`) -- the protobuf/gRPC contract
+    shared by both sides.
+
+By default the server listens on `127.0.0.1:5051`.
+
+## The key-value store
+
+A flat `string -> bytes` key-value store.
+
+| RPC | Description |
+| --- | --- |
+| `MapSet(key, value)` | Store `value` under `key`, overwriting any existing value. |
+| `MapGet(key)` | Return the value for `key`, or `NOT_FOUND` if it is missing. |
+
+Values are opaque bytes, so callers are free to store whatever serialization
+they like (JSON, pickle, protobuf, raw binary).
+
+## The task queue
+
+Tasks are units of work identified by a unique `task_id`.
+Each task carries an opaque `function` and `input` payload,
+a floating-point `priority`,
+and one or more named queues it should be dispatched from.
+A task moves through three states: `Ready` → `Running` → `Complete`.
+
+| RPC | Description |
+| --- | --- |
+| `TaskAdd(task_id, queue, priority, function, input)` | Register a new task and enqueue it on each named queue. Returns `ALREADY_EXISTS` if the id is already known. |
+| `TaskGet(worker_id, queue)` | Claim the highest-priority `Ready` task from the first non-empty queue, mark it `Running`, and return its payload. Returns `UNAVAILABLE` when no work is ready. |
+| `TaskDone(task_id, output)` | Mark a `Running` task `Complete` and store its output. |
+| `TaskStatus(task_id)` | Return a task's current state and, once complete, its output. |
+| `Requeue(timeout_s)` | Reset any task that has been `Running` longer than `timeout_s` back to `Ready` and re-enqueue it. |
+
+Within a queue, higher `priority` values are dispatched first.
+A worker polls `TaskGet` across the queues it cares about, runs the work,
+and reports back with `TaskDone`.
+`Requeue` provides fault tolerance:
+if a worker crashes without completing its task,
+a periodic `Requeue` call makes that task available to another worker.
+
+## Building the server
+
+Dependencies are managed with [Conan](https://conan.io/)
+and the build is driven by CMake.
+
+```sh
+conan install .
+. build/Release/generators/conanbuild.sh
+cmake -S . -B build/Release \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_TOOLCHAIN_FILE=generators/conan_toolchain.cmake
+cmake --build build/Release --parallel
+```
+
+A reproducible container build is defined in
+`scripts/apptainer/ds-server.def`.
+
+### Running
+
+```sh
+ds-server --address 0.0.0.0:5051
+```
+
+Run `ds-server --help` for the full argument list.
+
+## Python client
+
+```sh
+pip install .  # installs the ds-service-client package
+```
+
+```python
+from ds_service_client import Client, TaskState
+
+client = Client("127.0.0.1:5051")  # or set DS_SERVER_ADDRESS and call Client()
+
+# Key-value map
+client.map_set("greeting", b"hello")
+assert client.map_get("greeting") == b"hello"
+
+# Task queue
+client.task_add("job-1", queue="work", priority=1.0, function=b"...", input=b"...")
+
+task = client.task_get(worker_id="worker-a", queue="work")
+# ... do the work ...
+client.task_done(task.task_id, output=b"result")
+
+status = client.task_status("job-1")
+assert status.state == TaskState.Complete
+```
+
+If `Client()` is constructed without an address, it reads the server address
+from the `DS_SERVER_ADDRESS` environment variable.
+
+## License
+
+MIT -- see [LICENSE](LICENSE).

ds_service_client-0.2.0/conanfile.py

@@ -0,0 +1,49 @@
+# type: ignore
+from conan import ConanFile
+from conan.tools.cmake import cmake_layout, CMakeDeps, CMakeToolchain, CMake
+
+
+class DsServiceRecipe(ConanFile):
+    name = "ds-service"
+    version = "main"
+
+    settings = "os", "compiler", "build_type", "arch"
+    options = {"shared": [True, False], "fPIC": [True, False]}
+    default_options = {"shared": False, "fPIC": True}
+
+    exports_sources = "CMakeLists.txt", "cpp/*", "misc/*"
+
+    def layout(self):
+        cmake_layout(self)
+
+    def requirements(self):
+        self.requires("spdlog/1.17.0")
+        self.requires("argparse/3.2")
+        self.requires("parallel-hashmap/2.0.0")
+
+        self.requires("grpc/1.78.1")
+        self.requires("protobuf/6.33.5")
+
+    def build_requirements(self):
+        self.tool_requires("grpc/1.78.1")
+        self.tool_requires("protobuf/6.33.5")
+        self.tool_requires("cmake/4.3.0")
+
+    def generate(self):
+        deps = CMakeDeps(self)
+        deps.generate()
+
+        toolchain = CMakeToolchain(self)
+        toolchain.generate()
+
+    def build(self):
+        cmake = CMake(self)
+        cmake.configure()
+        cmake.build()
+
+    def package(self):
+        cmake = CMake(self)
+        cmake.install()
+
+    def package_info(self):
+        self.cpp_info.libs = ["ds-service-grpc"]