synapse-vault 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- synapse_vault-0.1.0/LICENSE +21 -0
- synapse_vault-0.1.0/PKG-INFO +152 -0
- synapse_vault-0.1.0/README.md +124 -0
- synapse_vault-0.1.0/pyproject.toml +51 -0
- synapse_vault-0.1.0/setup.cfg +4 -0
- synapse_vault-0.1.0/src/synapse/__init__.py +7 -0
- synapse_vault-0.1.0/src/synapse/__main__.py +4 -0
- synapse_vault-0.1.0/src/synapse/adapters/__init__.py +70 -0
- synapse_vault-0.1.0/src/synapse/adapters/chatgpt.py +152 -0
- synapse_vault-0.1.0/src/synapse/adapters/files.py +115 -0
- synapse_vault-0.1.0/src/synapse/build.py +250 -0
- synapse_vault-0.1.0/src/synapse/cli.py +186 -0
- synapse_vault-0.1.0/src/synapse/config.py +39 -0
- synapse_vault-0.1.0/src/synapse/dashboard.html +64 -0
- synapse_vault-0.1.0/src/synapse/db.py +46 -0
- synapse_vault-0.1.0/src/synapse/demo/raw/demo/2026-01/project-note.md +7 -0
- synapse_vault-0.1.0/src/synapse/demo/raw/demo/2026-02/community-note.md +7 -0
- synapse_vault-0.1.0/src/synapse/demo/raw/demo/2026-03/talk-outline.md +7 -0
- synapse_vault-0.1.0/src/synapse/demo/raw/demo/2026-04/garden-log.md +7 -0
- synapse_vault-0.1.0/src/synapse/demo/raw/demo/2026-05/goals.md +7 -0
- synapse_vault-0.1.0/src/synapse/demo/wiki/2026-goals.md +14 -0
- synapse_vault-0.1.0/src/synapse/demo/wiki/accessibility.md +13 -0
- synapse_vault-0.1.0/src/synapse/demo/wiki/civic-tech-meetup.md +13 -0
- synapse_vault-0.1.0/src/synapse/demo/wiki/climate-tech.md +13 -0
- synapse_vault-0.1.0/src/synapse/demo/wiki/collaborators.md +13 -0
- synapse_vault-0.1.0/src/synapse/demo/wiki/community-workshops.md +15 -0
- synapse_vault-0.1.0/src/synapse/demo/wiki/cycling.md +13 -0
- synapse_vault-0.1.0/src/synapse/demo/wiki/data-storytelling.md +13 -0
- synapse_vault-0.1.0/src/synapse/demo/wiki/design-principles.md +13 -0
- synapse_vault-0.1.0/src/synapse/demo/wiki/energy-visualization.md +13 -0
- synapse_vault-0.1.0/src/synapse/demo/wiki/helio.md +16 -0
- synapse_vault-0.1.0/src/synapse/demo/wiki/home-assistant.md +12 -0
- synapse_vault-0.1.0/src/synapse/demo/wiki/local-first-software.md +15 -0
- synapse_vault-0.1.0/src/synapse/demo/wiki/london.md +13 -0
- synapse_vault-0.1.0/src/synapse/demo/wiki/maya-chen.md +15 -0
- synapse_vault-0.1.0/src/synapse/demo/wiki/open-source.md +13 -0
- synapse_vault-0.1.0/src/synapse/demo/wiki/public-speaking.md +15 -0
- synapse_vault-0.1.0/src/synapse/demo/wiki/python.md +13 -0
- synapse_vault-0.1.0/src/synapse/demo/wiki/riverlight-garden.md +15 -0
- synapse_vault-0.1.0/src/synapse/demo/wiki/sqlite.md +15 -0
- synapse_vault-0.1.0/src/synapse/graph.py +107 -0
- synapse_vault-0.1.0/src/synapse/index.py +119 -0
- synapse_vault-0.1.0/src/synapse/ingest.py +77 -0
- synapse_vault-0.1.0/src/synapse/llm.py +84 -0
- synapse_vault-0.1.0/src/synapse/mcp.py +165 -0
- synapse_vault-0.1.0/src/synapse/models.py +22 -0
- synapse_vault-0.1.0/src/synapse/query.py +88 -0
- synapse_vault-0.1.0/src/synapse/raw.py +102 -0
- synapse_vault-0.1.0/src/synapse/server.py +264 -0
- synapse_vault-0.1.0/src/synapse/vault.py +65 -0
- synapse_vault-0.1.0/src/synapse_vault.egg-info/PKG-INFO +152 -0
- synapse_vault-0.1.0/src/synapse_vault.egg-info/SOURCES.txt +63 -0
- synapse_vault-0.1.0/src/synapse_vault.egg-info/dependency_links.txt +1 -0
- synapse_vault-0.1.0/src/synapse_vault.egg-info/entry_points.txt +2 -0
- synapse_vault-0.1.0/src/synapse_vault.egg-info/requires.txt +4 -0
- synapse_vault-0.1.0/src/synapse_vault.egg-info/top_level.txt +1 -0
- synapse_vault-0.1.0/tests/test_adapters.py +82 -0
- synapse_vault-0.1.0/tests/test_build.py +135 -0
- synapse_vault-0.1.0/tests/test_config.py +16 -0
- synapse_vault-0.1.0/tests/test_graph.py +49 -0
- synapse_vault-0.1.0/tests/test_ingest.py +38 -0
- synapse_vault-0.1.0/tests/test_mcp.py +67 -0
- synapse_vault-0.1.0/tests/test_query.py +44 -0
- synapse_vault-0.1.0/tests/test_server.py +52 -0
- synapse_vault-0.1.0/tests/test_vault.py +23 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Anshu Yadav
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: synapse-vault
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A local-first personal knowledge wiki backed by Markdown and SQLite
|
|
5
|
+
Author: Anshu Yadav
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/anshulyadav1976/synapse
|
|
8
|
+
Project-URL: Documentation, https://github.com/anshulyadav1976/synapse#readme
|
|
9
|
+
Project-URL: Issues, https://github.com/anshulyadav1976/synapse/issues
|
|
10
|
+
Keywords: knowledge-graph,local-first,markdown,mcp,personal-knowledge
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Environment :: Console
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: Database :: Database Engines/Servers
|
|
20
|
+
Classifier: Topic :: Text Processing :: Indexing
|
|
21
|
+
Requires-Python: >=3.11
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
Provides-Extra: dev
|
|
25
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
26
|
+
Requires-Dist: ruff>=0.6; extra == "dev"
|
|
27
|
+
Dynamic: license-file
|
|
28
|
+
|
|
29
|
+
<div align="center">
|
|
30
|
+
|
|
31
|
+
# Synapse
|
|
32
|
+
|
|
33
|
+

|
|
34
|
+
|
|
35
|
+
**Point it at text. Get a searchable archive for free. Spend a few cents turning it into a Markdown wiki your agent can query.**
|
|
36
|
+
|
|
37
|
+
[](https://github.com/anshulyadav1976/synapse/actions/workflows/ci.yml)
|
|
38
|
+
[](https://pypi.org/project/synapse-vault/)
|
|
39
|
+
[](https://www.python.org/)
|
|
40
|
+
[](https://github.com/anshulyadav1976/synapse/blob/main/pyproject.toml)
|
|
41
|
+
[](https://github.com/anshulyadav1976/synapse/blob/main/LICENSE)
|
|
42
|
+
|
|
43
|
+
</div>
|
|
44
|
+
|
|
45
|
+
Give Codex a private, local memory without filling every prompt with your history:
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
codex mcp add synapse -- uvx --from synapse-vault synapse mcp --vault /absolute/path/to/my-brain
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Nothing from the vault is injected into the agent's prompt. The agent searches first, then reads one relevant Markdown page. A thousand unopened pages cost zero tokens.
|
|
52
|
+
|
|
53
|
+
## See it in ten seconds
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
uvx --from synapse-vault synapse serve --demo
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
That opens a populated 20-page graph. It needs no API key, import, database server, or JavaScript build.
|
|
60
|
+
|
|
61
|
+
## Keep your own history
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
uvx --from synapse-vault synapse init ./my-brain
|
|
65
|
+
uvx --from synapse-vault synapse ingest ~/Downloads/chatgpt-export --vault ./my-brain
|
|
66
|
+
uvx --from synapse-vault synapse serve --vault ./my-brain
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
Ingest is local and free: it writes Markdown and builds a disposable SQLite FTS5 index. Search works immediately:
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
uvx --from synapse-vault synapse search "the phrase I remember" --vault ./my-brain
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
To turn raw history into a linked wiki, estimate first and then build a small resumable batch:
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
export SYNAPSE_API_KEY="..."
|
|
79
|
+
uvx --from synapse-vault synapse build --vault ./my-brain --limit 20 --dry-run
|
|
80
|
+
uvx --from synapse-vault synapse build --vault ./my-brain --limit 20
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
The build pass is optional. It is a plain `for` loop making one OpenAI-compatible chat-completions request per item. Run it against OpenAI or a local Ollama/LM Studio server; stop and resume without paying twice.
|
|
84
|
+
|
|
85
|
+
## Markdown is the database
|
|
86
|
+
|
|
87
|
+
```text
|
|
88
|
+
my-brain/
|
|
89
|
+
├── raw/<source>/<YYYY-MM>/<id>.md immutable imported history
|
|
90
|
+
├── wiki/<slug>.md linked, editable knowledge pages
|
|
91
|
+
├── synapse.db disposable FTS5 + graph index
|
|
92
|
+
└── synapse.toml model and owner settings
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Delete `synapse.db` and `synapse reindex --vault ./my-brain` recreates it. The durable data is ordinary Markdown that works with git, Obsidian, `grep`, and any editor.
|
|
96
|
+
|
|
97
|
+
## How it works
|
|
98
|
+
|
|
99
|
+
1. An adapter streams each source into a tiny `Item` shape with stable IDs and timestamps.
|
|
100
|
+
2. Ingest writes immutable raw Markdown and indexes it with SQLite FTS5—no model call.
|
|
101
|
+
3. Build compresses one item and asks any OpenAI-compatible model for complete wiki pages.
|
|
102
|
+
4. `[[wikilinks]]` become edges; a recursive SQLite CTE handles multi-hop traversal.
|
|
103
|
+
5. The one-file dashboard, CLI, Python API, REST API, and MCP server all use the same vault.
|
|
104
|
+
|
|
105
|
+
## Model providers
|
|
106
|
+
|
|
107
|
+
Synapse uses one standard-library HTTP POST to `/chat/completions`; there is no provider SDK.
|
|
108
|
+
|
|
109
|
+
| Provider | Base URL | Status |
|
|
110
|
+
|---|---|---|
|
|
111
|
+
| OpenAI | `https://api.openai.com/v1` | Tested with `gpt-4o-mini` |
|
|
112
|
+
| Ollama | `http://localhost:11434/v1` | Compatible; not yet in CI |
|
|
113
|
+
| LM Studio | `http://localhost:1234/v1` | Compatible; not yet in CI |
|
|
114
|
+
| OpenRouter | `https://openrouter.ai/api/v1` | Compatible; community verification wanted |
|
|
115
|
+
| Groq | `https://api.groq.com/openai/v1` | Compatible; community verification wanted |
|
|
116
|
+
| Together | `https://api.together.xyz/v1` | Compatible; community verification wanted |
|
|
117
|
+
| DeepSeek | `https://api.deepseek.com/v1` | Compatible; community verification wanted |
|
|
118
|
+
|
|
119
|
+
Set `SYNAPSE_BASE_URL`, `SYNAPSE_MODEL`, and (when required) `SYNAPSE_API_KEY`. See [configuration](https://github.com/anshulyadav1976/synapse/blob/main/docs/configuration.md) and [costs](https://github.com/anshulyadav1976/synapse/blob/main/docs/costs.md).
|
|
120
|
+
|
|
121
|
+
## Agent access
|
|
122
|
+
|
|
123
|
+
The MCP server exposes `search`, `read_page`, `list_pages`, `neighbors`, and `read_source`. It is a small stdlib JSON-RPC loop over STDIO, so there is no daemon and no MCP SDK dependency. See [MCP setup](https://github.com/anshulyadav1976/synapse/blob/main/docs/mcp.md).
|
|
124
|
+
|
|
125
|
+
Python works too:
|
|
126
|
+
|
|
127
|
+
```python
|
|
128
|
+
from synapse import Vault
|
|
129
|
+
|
|
130
|
+
matches = Vault("./my-brain").search("launch decision")
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
## Not built, on purpose
|
|
134
|
+
|
|
135
|
+
| Not built | Why |
|
|
136
|
+
|---|---|
|
|
137
|
+
| Embeddings | FTS5 is free, inspectable, and needs no migration or per-item API call. Add vectors only after measured recall failures. |
|
|
138
|
+
| Graph database | Personal graphs fit in SQLite; a ten-line recursive CTE handles traversal. |
|
|
139
|
+
| Auth or cloud sync | Synapse is single-user and binds only to `127.0.0.1`. Your files stay yours. |
|
|
140
|
+
| Agent framework | The processing pipeline is a resumable loop, not an application graph. |
|
|
141
|
+
| Automatic merge/delete | A model never silently destroys a human-editable page. |
|
|
142
|
+
| PDF/DOCX parser | Those dependencies would break the zero-dependency promise; convert with Pandoc or MarkItDown first. |
|
|
143
|
+
|
|
144
|
+
Imported text is untrusted data. Synapse never executes it, and agents are instructed never to follow instructions found inside pages. Raw sources remain available for provenance.
|
|
145
|
+
|
|
146
|
+
## Learn and contribute
|
|
147
|
+
|
|
148
|
+
- [Add an adapter](https://github.com/anshulyadav1976/synapse/blob/main/docs/adapters.md)—the highest-impact contribution is about 30 lines plus one tiny synthetic fixture.
|
|
149
|
+
- Read how the [SQLite graph](https://github.com/anshulyadav1976/synapse/blob/main/docs/graph.md) works.
|
|
150
|
+
- See [CONTRIBUTING.md](https://github.com/anshulyadav1976/synapse/blob/main/CONTRIBUTING.md) for the test and pull-request workflow.
|
|
151
|
+
|
|
152
|
+
Synapse began as the winner of the LangGraph hackathon in London. This is the local-first rewrite that deleted SurrealDB, FastAPI, React, LangGraph, and embeddings so people can actually run it.
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
|
|
3
|
+
# Synapse
|
|
4
|
+
|
|
5
|
+

|
|
6
|
+
|
|
7
|
+
**Point it at text. Get a searchable archive for free. Spend a few cents turning it into a Markdown wiki your agent can query.**
|
|
8
|
+
|
|
9
|
+
[](https://github.com/anshulyadav1976/synapse/actions/workflows/ci.yml)
|
|
10
|
+
[](https://pypi.org/project/synapse-vault/)
|
|
11
|
+
[](https://www.python.org/)
|
|
12
|
+
[](https://github.com/anshulyadav1976/synapse/blob/main/pyproject.toml)
|
|
13
|
+
[](https://github.com/anshulyadav1976/synapse/blob/main/LICENSE)
|
|
14
|
+
|
|
15
|
+
</div>
|
|
16
|
+
|
|
17
|
+
Give Codex a private, local memory without filling every prompt with your history:
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
codex mcp add synapse -- uvx --from synapse-vault synapse mcp --vault /absolute/path/to/my-brain
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
Nothing from the vault is injected into the agent's prompt. The agent searches first, then reads one relevant Markdown page. A thousand unopened pages cost zero tokens.
|
|
24
|
+
|
|
25
|
+
## See it in ten seconds
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
uvx --from synapse-vault synapse serve --demo
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
That opens a populated 20-page graph. It needs no API key, import, database server, or JavaScript build.
|
|
32
|
+
|
|
33
|
+
## Keep your own history
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
uvx --from synapse-vault synapse init ./my-brain
|
|
37
|
+
uvx --from synapse-vault synapse ingest ~/Downloads/chatgpt-export --vault ./my-brain
|
|
38
|
+
uvx --from synapse-vault synapse serve --vault ./my-brain
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Ingest is local and free: it writes Markdown and builds a disposable SQLite FTS5 index. Search works immediately:
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
uvx --from synapse-vault synapse search "the phrase I remember" --vault ./my-brain
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
To turn raw history into a linked wiki, estimate first and then build a small resumable batch:
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
export SYNAPSE_API_KEY="..."
|
|
51
|
+
uvx --from synapse-vault synapse build --vault ./my-brain --limit 20 --dry-run
|
|
52
|
+
uvx --from synapse-vault synapse build --vault ./my-brain --limit 20
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
The build pass is optional. It is a plain `for` loop making one OpenAI-compatible chat-completions request per item. Run it against OpenAI or a local Ollama/LM Studio server; stop and resume without paying twice.
|
|
56
|
+
|
|
57
|
+
## Markdown is the database
|
|
58
|
+
|
|
59
|
+
```text
|
|
60
|
+
my-brain/
|
|
61
|
+
├── raw/<source>/<YYYY-MM>/<id>.md immutable imported history
|
|
62
|
+
├── wiki/<slug>.md linked, editable knowledge pages
|
|
63
|
+
├── synapse.db disposable FTS5 + graph index
|
|
64
|
+
└── synapse.toml model and owner settings
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Delete `synapse.db` and `synapse reindex --vault ./my-brain` recreates it. The durable data is ordinary Markdown that works with git, Obsidian, `grep`, and any editor.
|
|
68
|
+
|
|
69
|
+
## How it works
|
|
70
|
+
|
|
71
|
+
1. An adapter streams each source into a tiny `Item` shape with stable IDs and timestamps.
|
|
72
|
+
2. Ingest writes immutable raw Markdown and indexes it with SQLite FTS5—no model call.
|
|
73
|
+
3. Build compresses one item and asks any OpenAI-compatible model for complete wiki pages.
|
|
74
|
+
4. `[[wikilinks]]` become edges; a recursive SQLite CTE handles multi-hop traversal.
|
|
75
|
+
5. The one-file dashboard, CLI, Python API, REST API, and MCP server all use the same vault.
|
|
76
|
+
|
|
77
|
+
## Model providers
|
|
78
|
+
|
|
79
|
+
Synapse uses one standard-library HTTP POST to `/chat/completions`; there is no provider SDK.
|
|
80
|
+
|
|
81
|
+
| Provider | Base URL | Status |
|
|
82
|
+
|---|---|---|
|
|
83
|
+
| OpenAI | `https://api.openai.com/v1` | Tested with `gpt-4o-mini` |
|
|
84
|
+
| Ollama | `http://localhost:11434/v1` | Compatible; not yet in CI |
|
|
85
|
+
| LM Studio | `http://localhost:1234/v1` | Compatible; not yet in CI |
|
|
86
|
+
| OpenRouter | `https://openrouter.ai/api/v1` | Compatible; community verification wanted |
|
|
87
|
+
| Groq | `https://api.groq.com/openai/v1` | Compatible; community verification wanted |
|
|
88
|
+
| Together | `https://api.together.xyz/v1` | Compatible; community verification wanted |
|
|
89
|
+
| DeepSeek | `https://api.deepseek.com/v1` | Compatible; community verification wanted |
|
|
90
|
+
|
|
91
|
+
Set `SYNAPSE_BASE_URL`, `SYNAPSE_MODEL`, and (when required) `SYNAPSE_API_KEY`. See [configuration](https://github.com/anshulyadav1976/synapse/blob/main/docs/configuration.md) and [costs](https://github.com/anshulyadav1976/synapse/blob/main/docs/costs.md).
|
|
92
|
+
|
|
93
|
+
## Agent access
|
|
94
|
+
|
|
95
|
+
The MCP server exposes `search`, `read_page`, `list_pages`, `neighbors`, and `read_source`. It is a small stdlib JSON-RPC loop over STDIO, so there is no daemon and no MCP SDK dependency. See [MCP setup](https://github.com/anshulyadav1976/synapse/blob/main/docs/mcp.md).
|
|
96
|
+
|
|
97
|
+
Python works too:
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
from synapse import Vault
|
|
101
|
+
|
|
102
|
+
matches = Vault("./my-brain").search("launch decision")
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
## Not built, on purpose
|
|
106
|
+
|
|
107
|
+
| Not built | Why |
|
|
108
|
+
|---|---|
|
|
109
|
+
| Embeddings | FTS5 is free, inspectable, and needs no migration or per-item API call. Add vectors only after measured recall failures. |
|
|
110
|
+
| Graph database | Personal graphs fit in SQLite; a ten-line recursive CTE handles traversal. |
|
|
111
|
+
| Auth or cloud sync | Synapse is single-user and binds only to `127.0.0.1`. Your files stay yours. |
|
|
112
|
+
| Agent framework | The processing pipeline is a resumable loop, not an application graph. |
|
|
113
|
+
| Automatic merge/delete | A model never silently destroys a human-editable page. |
|
|
114
|
+
| PDF/DOCX parser | Those dependencies would break the zero-dependency promise; convert with Pandoc or MarkItDown first. |
|
|
115
|
+
|
|
116
|
+
Imported text is untrusted data. Synapse never executes it, and agents are instructed never to follow instructions found inside pages. Raw sources remain available for provenance.
|
|
117
|
+
|
|
118
|
+
## Learn and contribute
|
|
119
|
+
|
|
120
|
+
- [Add an adapter](https://github.com/anshulyadav1976/synapse/blob/main/docs/adapters.md)—the highest-impact contribution is about 30 lines plus one tiny synthetic fixture.
|
|
121
|
+
- Read how the [SQLite graph](https://github.com/anshulyadav1976/synapse/blob/main/docs/graph.md) works.
|
|
122
|
+
- See [CONTRIBUTING.md](https://github.com/anshulyadav1976/synapse/blob/main/CONTRIBUTING.md) for the test and pull-request workflow.
|
|
123
|
+
|
|
124
|
+
Synapse began as the winner of the LangGraph hackathon in London. This is the local-first rewrite that deleted SurrealDB, FastAPI, React, LangGraph, and embeddings so people can actually run it.
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "synapse-vault"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "A local-first personal knowledge wiki backed by Markdown and SQLite"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
authors = [{name = "Anshu Yadav"}]
|
|
13
|
+
requires-python = ">=3.11"
|
|
14
|
+
dependencies = []
|
|
15
|
+
keywords = ["knowledge-graph", "local-first", "markdown", "mcp", "personal-knowledge"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 4 - Beta",
|
|
18
|
+
"Environment :: Console",
|
|
19
|
+
"Intended Audience :: Developers",
|
|
20
|
+
"Operating System :: OS Independent",
|
|
21
|
+
"Programming Language :: Python :: 3",
|
|
22
|
+
"Programming Language :: Python :: 3.11",
|
|
23
|
+
"Programming Language :: Python :: 3.12",
|
|
24
|
+
"Programming Language :: Python :: 3.13",
|
|
25
|
+
"Topic :: Database :: Database Engines/Servers",
|
|
26
|
+
"Topic :: Text Processing :: Indexing",
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
[project.urls]
|
|
30
|
+
Homepage = "https://github.com/anshulyadav1976/synapse"
|
|
31
|
+
Documentation = "https://github.com/anshulyadav1976/synapse#readme"
|
|
32
|
+
Issues = "https://github.com/anshulyadav1976/synapse/issues"
|
|
33
|
+
|
|
34
|
+
[project.optional-dependencies]
|
|
35
|
+
dev = ["pytest>=8", "ruff>=0.6"]
|
|
36
|
+
|
|
37
|
+
[project.scripts]
|
|
38
|
+
synapse = "synapse.cli:main"
|
|
39
|
+
|
|
40
|
+
[tool.setuptools.packages.find]
|
|
41
|
+
where = ["src"]
|
|
42
|
+
|
|
43
|
+
[tool.setuptools.package-data]
|
|
44
|
+
synapse = ["dashboard.html", "demo/wiki/*.md", "demo/raw/demo/*/*.md"]
|
|
45
|
+
|
|
46
|
+
[tool.pytest.ini_options]
|
|
47
|
+
pythonpath = ["src"]
|
|
48
|
+
|
|
49
|
+
[tool.ruff]
|
|
50
|
+
line-length = 100
|
|
51
|
+
target-version = "py311"
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""Small reader registry: every input format stops at Item."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import Callable, Iterator
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import TypeAlias
|
|
6
|
+
from zipfile import BadZipFile, ZipFile
|
|
7
|
+
|
|
8
|
+
from ..models import Item
|
|
9
|
+
|
|
10
|
+
Reader: TypeAlias = Callable[[Path], Iterator[Item]]
|
|
11
|
+
READERS: dict[str, Reader] = {}
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def adapter(name: str) -> Callable[[Reader], Reader]:
|
|
15
|
+
def register(reader: Reader) -> Reader:
|
|
16
|
+
READERS[name] = reader
|
|
17
|
+
return reader
|
|
18
|
+
|
|
19
|
+
return register
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _zip_names(path: Path) -> list[str]:
|
|
23
|
+
try:
|
|
24
|
+
with ZipFile(path) as archive:
|
|
25
|
+
return archive.namelist()
|
|
26
|
+
except (BadZipFile, OSError):
|
|
27
|
+
return []
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def detect(path: Path) -> str:
|
|
31
|
+
if path.is_dir():
|
|
32
|
+
names = {child.name for child in path.iterdir()}
|
|
33
|
+
if "export_manifest.json" in names or any(
|
|
34
|
+
name.startswith("conversations-") and name.endswith(".json") for name in names
|
|
35
|
+
):
|
|
36
|
+
return "chatgpt"
|
|
37
|
+
return "dir"
|
|
38
|
+
if path.name == "conversations.json" or (
|
|
39
|
+
path.name.startswith("conversations-") and path.suffix == ".json"
|
|
40
|
+
):
|
|
41
|
+
return "chatgpt"
|
|
42
|
+
if path.name == "chat.html":
|
|
43
|
+
return "chatgpt"
|
|
44
|
+
if path.suffix.lower() == ".zip":
|
|
45
|
+
names = _zip_names(path)
|
|
46
|
+
if any(
|
|
47
|
+
Path(name).name == "export_manifest.json"
|
|
48
|
+
or Path(name).name == "conversations.json"
|
|
49
|
+
or Path(name).name == "chat.html"
|
|
50
|
+
or (Path(name).name.startswith("conversations-") and name.endswith(".json"))
|
|
51
|
+
for name in names
|
|
52
|
+
):
|
|
53
|
+
return "chatgpt"
|
|
54
|
+
return "file"
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def read(path: str | Path, format_name: str | None = None) -> Iterator[Item]:
|
|
58
|
+
source = Path(path).expanduser().resolve()
|
|
59
|
+
name = format_name or detect(source)
|
|
60
|
+
try:
|
|
61
|
+
reader = READERS[name]
|
|
62
|
+
except KeyError as error:
|
|
63
|
+
choices = ", ".join(sorted(READERS))
|
|
64
|
+
raise ValueError(f"Unknown format {name!r}. Available formats: {choices}") from error
|
|
65
|
+
yield from reader(source)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
# Importing registers the built-in readers while keeping contributor adapters tiny.
|
|
69
|
+
from . import chatgpt, files # noqa: F401
|
|
70
|
+
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
"""Reader for modern and legacy ChatGPT exports."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import mmap
|
|
5
|
+
import re
|
|
6
|
+
from collections.abc import Iterable, Iterator
|
|
7
|
+
from datetime import UTC, datetime
|
|
8
|
+
from pathlib import Path, PurePosixPath
|
|
9
|
+
from typing import Any
|
|
10
|
+
from zipfile import ZipFile
|
|
11
|
+
|
|
12
|
+
from ..models import Item, Turn
|
|
13
|
+
from . import adapter
|
|
14
|
+
|
|
15
|
+
SHARD = re.compile(r"conversations-\d+\.json$")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _iso(value: object) -> str:
|
|
19
|
+
try:
|
|
20
|
+
return datetime.fromtimestamp(float(value), UTC).isoformat()
|
|
21
|
+
except (TypeError, ValueError, OSError):
|
|
22
|
+
return datetime.fromtimestamp(0, UTC).isoformat()
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _parts(content: dict[str, Any]) -> str:
|
|
26
|
+
return "\n".join(part for part in content.get("parts", []) if isinstance(part, str)).strip()
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _conversation(data: dict[str, Any]) -> Item:
|
|
30
|
+
messages: list[tuple[float, int, Turn]] = []
|
|
31
|
+
for order, node in enumerate(data.get("mapping", {}).values()):
|
|
32
|
+
message = node.get("message") or {}
|
|
33
|
+
text = _parts(message.get("content") or {})
|
|
34
|
+
if not text:
|
|
35
|
+
continue
|
|
36
|
+
author = message.get("author") or {}
|
|
37
|
+
role = str(author.get("role") or "unknown")
|
|
38
|
+
speaker = "me" if role == "user" else str(author.get("name") or role)
|
|
39
|
+
created = message.get("create_time")
|
|
40
|
+
messages.append((float(created or 0), order, Turn(speaker, text, _iso(created))))
|
|
41
|
+
messages.sort(key=lambda entry: (entry[0], entry[1]))
|
|
42
|
+
identifier = str(data.get("conversation_id") or data.get("id") or "")
|
|
43
|
+
if not identifier:
|
|
44
|
+
raise ValueError("A ChatGPT conversation is missing both conversation_id and id")
|
|
45
|
+
return Item(
|
|
46
|
+
id=identifier,
|
|
47
|
+
source="chatgpt",
|
|
48
|
+
title=str(data.get("title") or "Untitled conversation"),
|
|
49
|
+
ts=_iso(data.get("create_time")),
|
|
50
|
+
turns=[entry[2] for entry in messages],
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _items(conversations: Iterable[dict[str, Any]]) -> Iterator[Item]:
|
|
55
|
+
for conversation in conversations:
|
|
56
|
+
yield _conversation(conversation)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _manifest_files(data: dict[str, Any]) -> list[str]:
|
|
60
|
+
return [
|
|
61
|
+
str(entry["path"])
|
|
62
|
+
for entry in data.get("export_files", [])
|
|
63
|
+
if isinstance(entry, dict)
|
|
64
|
+
and "path" in entry
|
|
65
|
+
and (SHARD.search(str(entry["path"])) or PurePosixPath(str(entry["path"])).name == "conversations.json")
|
|
66
|
+
]
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _folder_files(path: Path) -> list[Path]:
|
|
70
|
+
manifest = path / "export_manifest.json"
|
|
71
|
+
if manifest.exists():
|
|
72
|
+
names = _manifest_files(json.loads(manifest.read_text(encoding="utf-8")))
|
|
73
|
+
found = [path / name for name in names if (path / name).is_file()]
|
|
74
|
+
if found:
|
|
75
|
+
return found
|
|
76
|
+
shards = sorted(path.glob("conversations-*.json"))
|
|
77
|
+
if shards:
|
|
78
|
+
return shards
|
|
79
|
+
legacy = path / "conversations.json"
|
|
80
|
+
return [legacy] if legacy.exists() else []
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _from_html_bytes(data: bytes, label: str) -> Iterator[Item]:
|
|
84
|
+
marker = b"var jsonData ="
|
|
85
|
+
start = data.find(marker)
|
|
86
|
+
if start < 0:
|
|
87
|
+
raise ValueError(f"No 'var jsonData =' conversation payload found in {label}")
|
|
88
|
+
start += len(marker)
|
|
89
|
+
end = data.find(b";</script>", start)
|
|
90
|
+
if end < 0:
|
|
91
|
+
end = data.find(b";\n", start)
|
|
92
|
+
if end < 0:
|
|
93
|
+
raise ValueError(f"The ChatGPT payload in {label} has no closing semicolon")
|
|
94
|
+
yield from _items(json.loads(data[start:end]))
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _from_html(path: Path) -> Iterator[Item]:
|
|
98
|
+
with path.open("rb") as handle, mmap.mmap(handle.fileno(), 0, access=mmap.ACCESS_READ) as data:
|
|
99
|
+
yield from _from_html_bytes(data, str(path))
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _from_zip(path: Path) -> Iterator[Item]:
|
|
103
|
+
with ZipFile(path) as archive:
|
|
104
|
+
names = archive.namelist()
|
|
105
|
+
manifest_name = next((name for name in names if PurePosixPath(name).name == "export_manifest.json"), None)
|
|
106
|
+
selected: list[str] = []
|
|
107
|
+
if manifest_name:
|
|
108
|
+
manifest = json.loads(archive.read(manifest_name))
|
|
109
|
+
base = PurePosixPath(manifest_name).parent
|
|
110
|
+
available = set(names)
|
|
111
|
+
selected = [str(base / name) for name in _manifest_files(manifest) if str(base / name) in available]
|
|
112
|
+
if not selected:
|
|
113
|
+
selected = sorted(
|
|
114
|
+
name
|
|
115
|
+
for name in names
|
|
116
|
+
if SHARD.search(PurePosixPath(name).name)
|
|
117
|
+
or PurePosixPath(name).name == "conversations.json"
|
|
118
|
+
)
|
|
119
|
+
if selected:
|
|
120
|
+
for name in selected:
|
|
121
|
+
yield from _items(json.loads(archive.read(name)))
|
|
122
|
+
return
|
|
123
|
+
html = next((name for name in names if PurePosixPath(name).name == "chat.html"), None)
|
|
124
|
+
if html:
|
|
125
|
+
yield from _from_html_bytes(archive.read(html), f"{path}!{html}")
|
|
126
|
+
return
|
|
127
|
+
raise ValueError(f"No ChatGPT conversations found in {path}")
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
@adapter("chatgpt")
|
|
131
|
+
def read_chatgpt(path: Path) -> Iterator[Item]:
|
|
132
|
+
if path.is_dir():
|
|
133
|
+
files = _folder_files(path)
|
|
134
|
+
if not files:
|
|
135
|
+
html = path / "chat.html"
|
|
136
|
+
if html.exists():
|
|
137
|
+
yield from _from_html(html)
|
|
138
|
+
return
|
|
139
|
+
raise ValueError(
|
|
140
|
+
f"No conversations found in {path}. Modern exports shard them as "
|
|
141
|
+
"conversations-000.json; pass --format chatgpt to force detection."
|
|
142
|
+
)
|
|
143
|
+
for file in files:
|
|
144
|
+
yield from _items(json.loads(file.read_text(encoding="utf-8")))
|
|
145
|
+
return
|
|
146
|
+
if path.suffix.lower() == ".zip":
|
|
147
|
+
yield from _from_zip(path)
|
|
148
|
+
elif path.name == "chat.html":
|
|
149
|
+
yield from _from_html(path)
|
|
150
|
+
else:
|
|
151
|
+
yield from _items(json.loads(path.read_text(encoding="utf-8")))
|
|
152
|
+
|