markdown-memory-vec 0.1.0__tar.gz

markdown_memory_vec-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,31 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+permissions:
+  id-token: write
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/markdown-memory-vec
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install build tools
+        run: pip install build
+
+      - name: Build package
+        run: python -m build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

markdown_memory_vec-0.1.0/.gitignore

@@ -0,0 +1,42 @@
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+*.egg
+.installed.cfg
+*.manifest
+*.spec
+pip-log.txt
+pip-delete-this-directory.txt
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+.pytype/
+.venv/
+env/
+venv/

markdown_memory_vec-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Aigente
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

markdown_memory_vec-0.1.0/PKG-INFO

@@ -0,0 +1,219 @@
+Metadata-Version: 2.4
+Name: markdown-memory-vec
+Version: 0.1.0
+Summary: Lightweight vector search for Markdown-based memory systems. Chunk, embed, index, and hybrid-search your .md knowledge base with sqlite-vec.
+Project-URL: Homepage, https://github.com/aigente/markdown-memory-vec
+Project-URL: Repository, https://github.com/aigente/markdown-memory-vec
+Author-email: Aigente <dev@aigente.io>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: RAG,embeddings,markdown,memory,semantic-search,sqlite-vec,vector-search
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Text Processing :: Indexing
+Classifier: Typing :: Typed
+Requires-Python: >=3.11
+Requires-Dist: pyyaml>=6.0
+Provides-Extra: dev
+Requires-Dist: mypy>=1.10.0; extra == 'dev'
+Requires-Dist: pyright>=1.1.380; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Requires-Dist: ruff>=0.4.0; extra == 'dev'
+Requires-Dist: types-pyyaml; extra == 'dev'
+Provides-Extra: vector
+Requires-Dist: sentence-transformers>=3.0; extra == 'vector'
+Requires-Dist: sqlite-vec>=0.1.6; extra == 'vector'
+Description-Content-Type: text/markdown
+
+# markdown-memory-vec
+
+Lightweight vector search for Markdown-based memory systems. Chunk, embed, index, and hybrid-search your `.md` knowledge base with sqlite-vec.
+
+<!-- Badges -->
+<!-- [![PyPI version](https://badge.fury.io/py/markdown-memory-vec.svg)](https://pypi.org/project/markdown-memory-vec/) -->
+<!-- [![Python](https://img.shields.io/pypi/pyversions/markdown-memory-vec.svg)](https://pypi.org/project/markdown-memory-vec/) -->
+<!-- [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) -->
+
+## Features
+
+- **Markdown-native**: YAML frontmatter parsing for metadata (importance, type, tags)
+- **Smart chunking**: Paragraph-aware splitting with configurable overlap (~400 tokens, 80 overlap)
+- **SHA-256 dedup**: Never re-embed unchanged content — incremental indexing is fast
+- **Hybrid search**: Combines semantic similarity (α), importance weighting (β), and temporal decay (γ)
+- **Zero-copy storage**: sqlite-vec KNN search with cosine distance in a single `.db` file
+
+## Quick Start
+
+### Installation
+
+```bash
+# Core only (YAML parsing, chunking, interfaces)
+pip install markdown-memory-vec
+
+# With vector search support (sqlite-vec + sentence-transformers)
+pip install 'markdown-memory-vec[vector]'
+```
+
+### Python API
+
+```python
+from memory_vec import MemoryVectorService
+
+svc = MemoryVectorService("/path/to/project")
+svc.rebuild_index()                        # Full index build
+results = svc.search("how to deploy")      # Hybrid search
+svc.close()
+```
+
+### Low-level API
+
+```python
+from memory_vec import (
+    SqliteVecStore,
+    SentenceTransformerEmbedder,
+    MemoryIndexer,
+    HybridSearchService,
+)
+
+store = SqliteVecStore("memory.db")
+store.ensure_tables()
+
+embedder = SentenceTransformerEmbedder()
+indexer = MemoryIndexer(store, embedder, memory_root="/path/to/memory")
+indexer.index_directory("/path/to/memory")
+
+search = HybridSearchService(vec_store=store, embedder=embedder)
+results = search.search("how to deploy")
+for r in results:
+    print(f"{r.file_path} (score={r.hybrid_score:.3f}): {r.chunk_text[:80]}...")
+```
+
+## CLI Usage
+
+```bash
+# Full rebuild
+memory-vec /path/to/project --rebuild
+
+# Incremental update (only changed files)
+memory-vec /path/to/project --incremental
+
+# Search
+memory-vec /path/to/project --search "how to deploy" --top-k 5
+
+# Statistics
+memory-vec /path/to/project --stats
+
+# Custom memory subdirectory
+memory-vec /path/to/project --rebuild --memory-subdir "docs/memory"
+
+# Verbose logging
+memory-vec /path/to/project --rebuild -v
+```
+
+## API Reference
+
+### High-level
+
+| Class | Description |
+|-------|-------------|
+| `MemoryVectorService` | All-in-one service: rebuild, incremental index, search, stats |
+
+### Components
+
+| Class | Description |
+|-------|-------------|
+| `SqliteVecStore` | sqlite-vec backed vector store with KNN search |
+| `SentenceTransformerEmbedder` | Lazy-loading sentence-transformers embedder |
+| `MemoryIndexer` | Markdown → chunks → embeddings → store pipeline |
+| `HybridSearchService` | Hybrid scoring: `α×semantic + β×importance + γ×temporal` |
+
+### Interfaces
+
+| Interface | Description |
+|-----------|-------------|
+| `IEmbedder` | Abstract embedder (`embed`, `embed_batch`, `dimension`) |
+| `ISqliteVecStore` | Abstract vector store (`add`, `search`, `delete`, `clear`, `count`) |
+
+### Data Types
+
+| Type | Description |
+|------|-------------|
+| `VectorRecord` | Record for insertion (id, embedding, metadata) |
+| `VectorSearchResult` | Raw KNN result (id, distance, metadata) |
+| `SearchResult` | Hybrid search result with all score components |
+| `MemoryVecMeta` | Metadata dataclass for stored embeddings |
+
+### Utilities
+
+| Function | Description |
+|----------|-------------|
+| `chunk_text(text, chunk_size, overlap_size)` | Split text into overlapping chunks |
+| `parse_frontmatter(text)` | Extract YAML frontmatter from Markdown |
+| `content_hash(text)` | SHA-256 hex digest |
+| `is_sqlite_vec_available()` | Check sqlite-vec availability |
+| `is_sentence_transformers_available()` | Check sentence-transformers availability |
+
+## Architecture
+
+```
+┌─────────────────────────────────────────────────────┐
+│                MemoryVectorService                  │
+│          (high-level orchestration layer)            │
+├──────────┬──────────┬──────────────┬────────────────┤
+│          │          │              │                 │
+│  Indexer  │  Search  │   Embedder   │     Store      │
+│          │          │              │                 │
+│ .md file │  hybrid  │  sentence-   │  sqlite-vec    │
+│ → chunks │  scoring │  transformers│  KNN + meta    │
+│ → embed  │  α+β+γ   │  (lazy load) │  (cosine)      │
+│ → store  │          │              │                 │
+└──────────┴──────────┴──────────────┴────────────────┘
+     ▲                                      │
+     │         YAML frontmatter             │
+     │         importance/type/tags          ▼
+   ┌─┴─────────────────────┐     ┌─────────────────────┐
+   │    Markdown Files     │     │   vector_index.db    │
+   │   (.claude/memory/)   │     │   (single file)      │
+   └───────────────────────┘     └─────────────────────┘
+```
+
+## Configuration
+
+### HuggingFace Model
+
+By default, uses `paraphrase-multilingual-MiniLM-L12-v2` (384-dim, 50+ languages).
+
+For users in China or regions with slow HuggingFace access:
+
+```bash
+# Use a mirror
+export HF_ENDPOINT=https://hf-mirror.com
+
+# Or use offline mode (model must be pre-cached)
+export HF_HUB_OFFLINE=1
+```
+
+### Hybrid Search Weights
+
+Default: `α=0.6, β=0.2, γ=0.2, λ=0.05`
+
+```python
+search = HybridSearchService(
+    vec_store=store,
+    embedder=embedder,
+    alpha=0.8,    # Semantic weight
+    beta=0.1,     # Importance weight
+    gamma=0.1,    # Temporal decay weight
+    decay_lambda=0.03,  # Slower decay
+)
+```
+
+## License
+
+MIT

markdown_memory_vec-0.1.0/README.md

@@ -0,0 +1,186 @@
+# markdown-memory-vec
+
+Lightweight vector search for Markdown-based memory systems. Chunk, embed, index, and hybrid-search your `.md` knowledge base with sqlite-vec.
+
+<!-- Badges -->
+<!-- [![PyPI version](https://badge.fury.io/py/markdown-memory-vec.svg)](https://pypi.org/project/markdown-memory-vec/) -->
+<!-- [![Python](https://img.shields.io/pypi/pyversions/markdown-memory-vec.svg)](https://pypi.org/project/markdown-memory-vec/) -->
+<!-- [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) -->
+
+## Features
+
+- **Markdown-native**: YAML frontmatter parsing for metadata (importance, type, tags)
+- **Smart chunking**: Paragraph-aware splitting with configurable overlap (~400 tokens, 80 overlap)
+- **SHA-256 dedup**: Never re-embed unchanged content — incremental indexing is fast
+- **Hybrid search**: Combines semantic similarity (α), importance weighting (β), and temporal decay (γ)
+- **Zero-copy storage**: sqlite-vec KNN search with cosine distance in a single `.db` file
+
+## Quick Start
+
+### Installation
+
+```bash
+# Core only (YAML parsing, chunking, interfaces)
+pip install markdown-memory-vec
+
+# With vector search support (sqlite-vec + sentence-transformers)
+pip install 'markdown-memory-vec[vector]'
+```
+
+### Python API
+
+```python
+from memory_vec import MemoryVectorService
+
+svc = MemoryVectorService("/path/to/project")
+svc.rebuild_index()                        # Full index build
+results = svc.search("how to deploy")      # Hybrid search
+svc.close()
+```
+
+### Low-level API
+
+```python
+from memory_vec import (
+    SqliteVecStore,
+    SentenceTransformerEmbedder,
+    MemoryIndexer,
+    HybridSearchService,
+)
+
+store = SqliteVecStore("memory.db")
+store.ensure_tables()
+
+embedder = SentenceTransformerEmbedder()
+indexer = MemoryIndexer(store, embedder, memory_root="/path/to/memory")
+indexer.index_directory("/path/to/memory")
+
+search = HybridSearchService(vec_store=store, embedder=embedder)
+results = search.search("how to deploy")
+for r in results:
+    print(f"{r.file_path} (score={r.hybrid_score:.3f}): {r.chunk_text[:80]}...")
+```
+
+## CLI Usage
+
+```bash
+# Full rebuild
+memory-vec /path/to/project --rebuild
+
+# Incremental update (only changed files)
+memory-vec /path/to/project --incremental
+
+# Search
+memory-vec /path/to/project --search "how to deploy" --top-k 5
+
+# Statistics
+memory-vec /path/to/project --stats
+
+# Custom memory subdirectory
+memory-vec /path/to/project --rebuild --memory-subdir "docs/memory"
+
+# Verbose logging
+memory-vec /path/to/project --rebuild -v
+```
+
+## API Reference
+
+### High-level
+
+| Class | Description |
+|-------|-------------|
+| `MemoryVectorService` | All-in-one service: rebuild, incremental index, search, stats |
+
+### Components
+
+| Class | Description |
+|-------|-------------|
+| `SqliteVecStore` | sqlite-vec backed vector store with KNN search |
+| `SentenceTransformerEmbedder` | Lazy-loading sentence-transformers embedder |
+| `MemoryIndexer` | Markdown → chunks → embeddings → store pipeline |
+| `HybridSearchService` | Hybrid scoring: `α×semantic + β×importance + γ×temporal` |
+
+### Interfaces
+
+| Interface | Description |
+|-----------|-------------|
+| `IEmbedder` | Abstract embedder (`embed`, `embed_batch`, `dimension`) |
+| `ISqliteVecStore` | Abstract vector store (`add`, `search`, `delete`, `clear`, `count`) |
+
+### Data Types
+
+| Type | Description |
+|------|-------------|
+| `VectorRecord` | Record for insertion (id, embedding, metadata) |
+| `VectorSearchResult` | Raw KNN result (id, distance, metadata) |
+| `SearchResult` | Hybrid search result with all score components |
+| `MemoryVecMeta` | Metadata dataclass for stored embeddings |
+
+### Utilities
+
+| Function | Description |
+|----------|-------------|
+| `chunk_text(text, chunk_size, overlap_size)` | Split text into overlapping chunks |
+| `parse_frontmatter(text)` | Extract YAML frontmatter from Markdown |
+| `content_hash(text)` | SHA-256 hex digest |
+| `is_sqlite_vec_available()` | Check sqlite-vec availability |
+| `is_sentence_transformers_available()` | Check sentence-transformers availability |
+
+## Architecture
+
+```
+┌─────────────────────────────────────────────────────┐
+│                MemoryVectorService                  │
+│          (high-level orchestration layer)            │
+├──────────┬──────────┬──────────────┬────────────────┤
+│          │          │              │                 │
+│  Indexer  │  Search  │   Embedder   │     Store      │
+│          │          │              │                 │
+│ .md file │  hybrid  │  sentence-   │  sqlite-vec    │
+│ → chunks │  scoring │  transformers│  KNN + meta    │
+│ → embed  │  α+β+γ   │  (lazy load) │  (cosine)      │
+│ → store  │          │              │                 │
+└──────────┴──────────┴──────────────┴────────────────┘
+     ▲                                      │
+     │         YAML frontmatter             │
+     │         importance/type/tags          ▼
+   ┌─┴─────────────────────┐     ┌─────────────────────┐
+   │    Markdown Files     │     │   vector_index.db    │
+   │   (.claude/memory/)   │     │   (single file)      │
+   └───────────────────────┘     └─────────────────────┘
+```
+
+## Configuration
+
+### HuggingFace Model
+
+By default, uses `paraphrase-multilingual-MiniLM-L12-v2` (384-dim, 50+ languages).
+
+For users in China or regions with slow HuggingFace access:
+
+```bash
+# Use a mirror
+export HF_ENDPOINT=https://hf-mirror.com
+
+# Or use offline mode (model must be pre-cached)
+export HF_HUB_OFFLINE=1
+```
+
+### Hybrid Search Weights
+
+Default: `α=0.6, β=0.2, γ=0.2, λ=0.05`
+
+```python
+search = HybridSearchService(
+    vec_store=store,
+    embedder=embedder,
+    alpha=0.8,    # Semantic weight
+    beta=0.1,     # Importance weight
+    gamma=0.1,    # Temporal decay weight
+    decay_lambda=0.03,  # Slower decay
+)
+```
+
+## License
+
+MIT

markdown_memory_vec-0.1.0/pyproject.toml

@@ -0,0 +1,143 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "markdown-memory-vec"
+version = "0.1.0"
+description = "Lightweight vector search for Markdown-based memory systems. Chunk, embed, index, and hybrid-search your .md knowledge base with sqlite-vec."
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.11"
+authors = [
+    { name = "Aigente", email = "dev@aigente.io" },
+]
+keywords = [
+    "vector-search",
+    "markdown",
+    "memory",
+    "embeddings",
+    "sqlite-vec",
+    "semantic-search",
+    "RAG",
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Text Processing :: Indexing",
+    "Typing :: Typed",
+]
+dependencies = [
+    "PyYAML>=6.0",
+]
+
+[project.optional-dependencies]
+vector = [
+    "sqlite-vec>=0.1.6",
+    "sentence-transformers>=3.0",
+]
+dev = [
+    "pytest>=7.0",
+    "pyright>=1.1.380",
+    "ruff>=0.4.0",
+    "mypy>=1.10.0",
+    "types-PyYAML",
+]
+
+[project.urls]
+Homepage = "https://github.com/aigente/markdown-memory-vec"
+Repository = "https://github.com/aigente/markdown-memory-vec"
+
+[project.scripts]
+memory-vec = "memory_vec.__main__:main"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/memory_vec"]
+
+# ---------------------------------------------------------------------------
+# Ruff
+# ---------------------------------------------------------------------------
+[tool.ruff]
+line-length = 120
+fix = true
+target-version = "py311"
+include = ["src/**", "tests/**"]
+
+[tool.ruff.format]
+docstring-code-format = true
+
+[tool.ruff.lint]
+select = [
+    "E",
+    "F",
+    "W",
+    "B",
+    "Q",
+    "I",
+    "ASYNC",
+    "T20",
+]
+ignore = [
+    "F401",
+    "E501",
+    "W293",
+    "W291",
+]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**" = ["T201"]  # Allow print statements in test files
+
+# ---------------------------------------------------------------------------
+# Pyright
+# ---------------------------------------------------------------------------
+[tool.pyright]
+include = ["src", "tests"]
+typeCheckingMode = "strict"
+reportUnnecessaryIsInstance = false
+reportMissingTypeStubs = false
+reportUnknownVariableType = false
+reportUnknownMemberType = false
+reportUnknownArgumentType = false
+reportUnusedImport = false
+
+# ---------------------------------------------------------------------------
+# Mypy
+# ---------------------------------------------------------------------------
+[tool.mypy]
+strict = true
+python_version = "3.11"
+ignore_missing_imports = true
+disallow_untyped_defs = true
+no_implicit_optional = true
+check_untyped_defs = true
+warn_return_any = true
+show_error_codes = true
+warn_unused_ignores = false
+disallow_incomplete_defs = true
+disallow_untyped_decorators = true
+disallow_any_unimported = false
+cache_dir = ".mypy_cache"
+
+# ---------------------------------------------------------------------------
+# Pytest
+# ---------------------------------------------------------------------------
+[tool.pytest.ini_options]
+minversion = "6.0"
+testpaths = ["tests"]
+
+# ---------------------------------------------------------------------------
+# Poe the Poet (task runner)
+# ---------------------------------------------------------------------------
+[tool.poe.tasks]
+fmt = "ruff format"
+lint = "ruff check"
+pyright = "pyright"
+mypy = "mypy src tests"
+test = "pytest -v"
+check = ["fmt", "lint", "pyright", "test"]