stixdb-engine 0.1.0__tar.gz

stixdb_engine-0.1.0/.env.example

@@ -0,0 +1,91 @@
+# ==========================================
+# StixDB Environment Configuration Example
+# ==========================================
+
+# ------------------------------------------
+# 1. Background Agent Configuration
+# ------------------------------------------
+STIXDB_AGENT_CYCLE_INTERVAL=30.0               # Seconds between perceive/plan/act loops
+STIXDB_AGENT_CONSOLIDATION_THRESHOLD=0.88      # Minimum cosine similarity to merge redundant nodes
+STIXDB_AGENT_DECAY_HALF_LIFE=48.0              # Hours before an unaccessed node's importance halves
+STIXDB_AGENT_PRUNE_THRESHOLD=0.00              # Node importance below this will be permanently pruned
+STIXDB_AGENT_WORKING_MEMORY_MAX=256            # Max number of hot nodes kept in working memory
+STIXDB_AGENT_MAX_CONSOLIDATION_BATCH=64        # Max nodes the regenerator merges per cycle
+STIXDB_AGENT_AUTO_SUMMARIZE=true               # Should agents automatically summarize large clusters?
+STIXDB_AGENT_LINEAGE_SAFE_MODE=true            # Preserve summarized source nodes instead of pruning them later
+
+# ------------------------------------------
+# 2. LLM Reasoner Configuration
+# ------------------------------------------
+STIXDB_LLM_PROVIDER=custom                     # openai | anthropic | ollama | custom | none
+STIXDB_LLM_MODEL=nvidia/nemotron-3-super-120b-a12b # E.g., gpt-4o, claude-3-5-sonnet, llama3, or custom model
+STIXDB_LLM_TEMPERATURE=0.2                     # Temp controlling generation parameters
+STIXDB_LLM_MAX_TOKENS=2048                     # Max context token sequence length
+STIXDB_LLM_MAX_CONTEXT_NODES=20                # Max number of graph nodes provided as context
+STIXDB_LLM_GRAPH_TRAVERSAL_DEPTH=3             # Max depth of graph traversal during query retrieval
+STIXDB_LLM_TIMEOUT=60.0                        # Rest API timeout for completion calls
+
+# ------------------------------------------
+# 3. Embedding Provider Configuration
+# ------------------------------------------
+STIXDB_EMBEDDING_PROVIDER=custom               # sentence_transformers | openai | ollama | custom
+STIXDB_EMBEDDING_MODEL=Qwen/Qwen3-Embedding-8B # Model architecture mapped via string name 
+STIXDB_EMBEDDING_DIMENSIONS=384                # Dimension size for embeddings generated
+
+# ------------------------------------------
+# 4. Storage & Vector DB Configuration
+# ------------------------------------------
+# STIXDB_STORAGE_MODE controls the graph backend:
+#
+#   memory  — In-process NetworkX (fastest, no persistence, data lost on restart)
+#   kuzu    — KuzuDB embedded (persistent on disk, no Docker required) ✔ RECOMMENDED FOR LOCAL DEV
+#   neo4j   — Neo4j via Docker (production)
+#
+STIXDB_STORAGE_MODE=kuzu                       # memory | kuzu | neo4j
+STIXDB_DATA_DIR=./stixdb_data                   # Root data directory
+STIXDB_KUZU_PATH=./stixdb_data/kuzu             # Path for KuzuDB files (kuzu mode only)
+STIXDB_VECTOR_BACKEND=memory                   # memory (in-proc) | chroma | qdrant
+STIXDB_STORAGE_MAX_ACTIVE_NODES=1000000        # Safety cap enforcing eviction threshold
+
+# ExtDB Endpoints (Required only if using Qdrant/Chroma)
+# QDRANT_HOST=localhost
+# QDRANT_PORT=6333
+# CHROMA_HOST=localhost
+
+# ------------------------------------------
+# 5. External API Keys & Endpoints
+# ------------------------------------------
+# OPENAI_API_KEY=sk-your-openai-key-here
+# ANTHROPIC_API_KEY=sk-ant-your-anthropic-key-here
+# OLLAMA_BASE_URL=http://localhost:11434
+
+# Custom OpenAI-compatible provider (only if STIXDB_LLM_PROVIDER=custom)
+# STIXDB_LLM_CUSTOM_BASE_URL=https://your-provider.com/v1/
+# STIXDB_LLM_CUSTOM_API_KEY=your-custom-llm-api-key
+
+# Custom embedding provider (only if STIXDB_EMBEDDING_PROVIDER=custom)
+# STIXDB_EMBEDDING_CUSTOM_BASE_URL=https://your-provider.com/v1/
+# STIXDB_EMBEDDING_CUSTOM_API_KEY=your-custom-embedding-api-key
+
+# ------------------------------------------
+# 6. Ingestion Configuration
+# ------------------------------------------
+STIXDB_CHUNK_SIZE=1000                       # Characters per ingestion chunk
+STIXDB_CHUNK_OVERLAP=200                     # Overlap between consecutive chunks
+                                              # Supported: .txt .md .pdf (native text) and all code/
+                                              # markup extensions. For scanned PDFs, run OCR externally
+                                              # and upload the resulting .md or .txt file.
+
+# ------------------------------------------
+# 7. Observability & Telemetry
+# ------------------------------------------
+STIXDB_ENABLE_TRACES=true                      # Enables distributed tracing
+STIXDB_ENABLE_METRICS=true                     # Enables Prometheus metrics monitoring
+STIXDB_METRICS_PORT=9090                       # Port Prometheus server exposes
+STIXDB_LOG_LEVEL=INFO                          # DEBUG, INFO, WARNING, ERROR
+
+# ------------------------------------------
+# 7. Server & API Configuration
+# ------------------------------------------
+STIXDB_API_PORT=4020                           # Port where the REST API runs
+STIXDB_API_KEY=your-secure-api-key-here        # If set, all requests must pass X-API-Key header

stixdb_engine-0.1.0/.gitattributes

@@ -0,0 +1,2 @@
+# Auto detect text files and perform LF normalization
+* text=auto

stixdb_engine-0.1.0/.gitignore

@@ -0,0 +1,71 @@
+# Python
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+*.so
+
+# Virtual environments
+.venv/
+venv/
+env/
+ENV/
+
+# Build & distribution
+*.egg-info/
+dist/
+build/
+*.egg
+MANIFEST
+*.whl
+
+# Pytest / coverage
+.pytest_cache/
+.coverage
+.coverage.*
+htmlcov/
+.hypothesis/
+
+# Mypy / type checkers
+.mypy_cache/
+.dmypy.json
+dmypy.json
+.pyre/
+
+# Ruff
+.ruff_cache/
+
+# Environment files — NEVER commit real credentials
+.env
+.env.local
+.env.*.local
+stix/.env
+
+# IDE / editor
+.vscode/
+.idea/
+*.sublime-project
+*.sublime-workspace
+.DS_Store
+Thumbs.db
+
+# Claude Code local settings
+.claude/
+
+# STIX runtime data
+stix_data/
+stix/stix_data/
+*.db
+
+# Logs
+*.log
+logs/
+
+# Notebooks checkpoints
+.ipynb_checkpoints/
+
+# Docker volumes (local dev)
+neo4j_data/
+chroma_data/
+pg_data/
+

stixdb_engine-0.1.0/CHANGELOG.md

@@ -0,0 +1,49 @@
+# Changelog
+
+All notable changes to StixDB are documented here.
+
+The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
+StixDB uses [Semantic Versioning](https://semver.org/).
+
+---
+
+## [Unreleased]
+
+### Planned
+- Pinecone vector backend
+- Multi-hop graph reasoning traces UI
+- Collection-level RBAC
+- Webhook callbacks on agent cycle events
+
+---
+
+## [0.1.0] — 2025-01-01
+
+### Added
+- **StixDBEngine** — top-level async engine managing multiple isolated collections
+- **MemoryGraph** — unified graph + vector store interface (NetworkX / KuzuDB / Neo4j)
+- **MemoryAgent** — per-collection autonomous background agent
+  - `AccessPlanner` — hybrid LRU+LFU heat scoring for node tier promotion
+  - `Consolidator` — cosine-similarity merging (threshold 0.88) + exponential decay pruning
+  - `MemoryAgentWorker` — decoupled `perceive → plan → act` async loop (30s default)
+- **ContextBroker** — 7-phase retrieval: embed → vector search → graph BFS → re-rank → truncate → LLM reason → record
+- **Reasoner** — LLM synthesis over graph context (OpenAI / Anthropic / Ollama / Custom / None)
+- **REST API** (FastAPI)
+  - Collection CRUD, bulk ingest, file upload
+  - `POST /collections/{id}/ask` — agentic Q&A
+  - `POST /search` — multi-query, cross-collection, filterable search
+  - `GET /collections/{id}/agent/status` — agent introspection
+  - `GET /traces` — execution trace log
+  - OpenAI-compatible `/v1/chat/completions`, `/v1/models`, `/v1/embeddings`
+- **Python SDK** (`stixdb-sdk`) — sync + async HTTP client
+- **Storage backends**: NetworkX (ephemeral), KuzuDB (local persistent, no Docker), Neo4j (Docker, production)
+- **Vector backends**: NumPy (in-process), ChromaDB, Qdrant
+- **Embedding providers**: sentence-transformers, OpenAI, Ollama, custom OpenAI-compatible
+- **Memory tiers**: `working`, `episodic`, `semantic`, `procedural`, `archived`
+- **Node types**: `fact`, `entity`, `event`, `concept`, `procedure`, `summary`, `question`
+- **Lineage safety mode** — source nodes pinned across consolidation cycles
+- **Document ingestion** — PDF (page-level provenance), plain text (character offset chunking)
+- **Observability** — structlog, Prometheus metrics, distributed trace log
+- **Docker Compose** stack — StixDB + Neo4j + ChromaDB + PostgreSQL
+- **CLI** — `stixdb serve`, `stixdb demo`, `stixdb multi-demo`
+- Full test suite — agent, graph, lineage, search API, OpenAI compatibility, SDK

stixdb_engine-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,123 @@
+# Contributing to StixDB
+
+Thanks for your interest — all contributions are welcome: bug reports, features, documentation, examples, and storage/embedding backend integrations.
+
+---
+
+## Getting Started
+
+### Prerequisites
+
+- Python 3.10+
+- Git
+
+### Setup
+
+```bash
+git clone https://github.com/your-org/stix.git
+cd stix
+
+python -m venv .venv
+source .venv/bin/activate      # Windows: .venv\Scripts\activate
+
+pip install -e ".[dev]"
+```
+
+Copy the example env and configure it:
+
+```bash
+cp .env.example .env
+# Edit .env — at minimum set STIXDB_LLM_PROVIDER=none for testing without an API key
+```
+
+### Run the tests
+
+```bash
+pytest tests/ -v
+```
+
+Tests run in heuristic mode (`LLMProvider.NONE`) by default — no API key needed.
+
+---
+
+## Project Structure
+
+```
+stix/               Core engine
+  agent/            MemoryAgent — AccessPlanner, Consolidator, Worker
+  api/              FastAPI server and routes
+  context/          ContextBroker (7-phase retrieval) and Reasoner
+  graph/            MemoryGraph, node/edge/cluster models
+  storage/          StorageBackend implementations (NetworkX, KuzuDB, Neo4j)
+                    VectorStore implementations (NumPy, ChromaDB, Qdrant)
+                    EmbeddingClient (sentence-transformers, OpenAI, Ollama, custom)
+  ingestion/        Document parsing and chunking
+  observability/    Structured logging and trace emission
+
+sdk/                Python HTTP client (stixdb-sdk)
+  src/stixdb_sdk/     Client, MemoryAPI, QueryAPI, SearchAPI
+
+examples/           Runnable examples
+tests/              Automated test suite
+doc/                Architecture and performance documentation
+```
+
+---
+
+## How to Contribute
+
+### Reporting a bug
+
+Open a [GitHub Issue](https://github.com/your-org/stix/issues/new?template=bug_report.md) with:
+- Reproduction steps (minimal code)
+- Your environment (Python version, storage/vector backend, OS)
+- Full stack trace
+
+### Requesting a feature
+
+Open a [GitHub Issue](https://github.com/your-org/stix/issues/new?template=feature_request.md) describing the problem and your proposed solution.
+
+### Submitting a pull request
+
+1. Fork the repo and create a branch: `feature/my-thing` or `fix/issue-123`
+2. Make your changes
+3. Add or update tests in `tests/`
+4. Run `pytest tests/ -v` — all tests must pass
+5. Run `ruff check stix/ sdk/src/ && ruff format --check stix/ sdk/src/`
+6. Open a PR with a clear description (use the PR template)
+
+---
+
+## Adding a New Storage Backend
+
+Implement `stix/storage/base.py:StorageBackend` and register it in `stix/config.py` under `StorageMode`. See `stix/storage/networkx_backend.py` for the simplest reference implementation.
+
+## Adding a New Vector Backend
+
+Implement the `VectorStore` protocol in `stix/storage/vector_store.py`. See `MemoryVectorStore` for a minimal reference.
+
+## Adding a New LLM Provider
+
+Extend `stix/agent/reasoner.py` — add a branch in `Reasoner._call_llm` and register the new value in `LLMProvider`.
+
+---
+
+## Code Style
+
+- Formatter: `ruff format` (line length 100)
+- Linter: `ruff check`
+- Logging: `structlog` — never use `print()` in library code
+- Models: Pydantic v2
+- Tests: `pytest` + `pytest-asyncio`
+
+---
+
+## Security
+
+Do not open public issues for security vulnerabilities. See [SECURITY.md](SECURITY.md).
+
+---
+
+## License
+
+By contributing you agree that your contributions are licensed under the [MIT License](LICENSE).