synapse-vault 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. synapse_vault-0.1.0/LICENSE +21 -0
  2. synapse_vault-0.1.0/PKG-INFO +152 -0
  3. synapse_vault-0.1.0/README.md +124 -0
  4. synapse_vault-0.1.0/pyproject.toml +51 -0
  5. synapse_vault-0.1.0/setup.cfg +4 -0
  6. synapse_vault-0.1.0/src/synapse/__init__.py +7 -0
  7. synapse_vault-0.1.0/src/synapse/__main__.py +4 -0
  8. synapse_vault-0.1.0/src/synapse/adapters/__init__.py +70 -0
  9. synapse_vault-0.1.0/src/synapse/adapters/chatgpt.py +152 -0
  10. synapse_vault-0.1.0/src/synapse/adapters/files.py +115 -0
  11. synapse_vault-0.1.0/src/synapse/build.py +250 -0
  12. synapse_vault-0.1.0/src/synapse/cli.py +186 -0
  13. synapse_vault-0.1.0/src/synapse/config.py +39 -0
  14. synapse_vault-0.1.0/src/synapse/dashboard.html +64 -0
  15. synapse_vault-0.1.0/src/synapse/db.py +46 -0
  16. synapse_vault-0.1.0/src/synapse/demo/raw/demo/2026-01/project-note.md +7 -0
  17. synapse_vault-0.1.0/src/synapse/demo/raw/demo/2026-02/community-note.md +7 -0
  18. synapse_vault-0.1.0/src/synapse/demo/raw/demo/2026-03/talk-outline.md +7 -0
  19. synapse_vault-0.1.0/src/synapse/demo/raw/demo/2026-04/garden-log.md +7 -0
  20. synapse_vault-0.1.0/src/synapse/demo/raw/demo/2026-05/goals.md +7 -0
  21. synapse_vault-0.1.0/src/synapse/demo/wiki/2026-goals.md +14 -0
  22. synapse_vault-0.1.0/src/synapse/demo/wiki/accessibility.md +13 -0
  23. synapse_vault-0.1.0/src/synapse/demo/wiki/civic-tech-meetup.md +13 -0
  24. synapse_vault-0.1.0/src/synapse/demo/wiki/climate-tech.md +13 -0
  25. synapse_vault-0.1.0/src/synapse/demo/wiki/collaborators.md +13 -0
  26. synapse_vault-0.1.0/src/synapse/demo/wiki/community-workshops.md +15 -0
  27. synapse_vault-0.1.0/src/synapse/demo/wiki/cycling.md +13 -0
  28. synapse_vault-0.1.0/src/synapse/demo/wiki/data-storytelling.md +13 -0
  29. synapse_vault-0.1.0/src/synapse/demo/wiki/design-principles.md +13 -0
  30. synapse_vault-0.1.0/src/synapse/demo/wiki/energy-visualization.md +13 -0
  31. synapse_vault-0.1.0/src/synapse/demo/wiki/helio.md +16 -0
  32. synapse_vault-0.1.0/src/synapse/demo/wiki/home-assistant.md +12 -0
  33. synapse_vault-0.1.0/src/synapse/demo/wiki/local-first-software.md +15 -0
  34. synapse_vault-0.1.0/src/synapse/demo/wiki/london.md +13 -0
  35. synapse_vault-0.1.0/src/synapse/demo/wiki/maya-chen.md +15 -0
  36. synapse_vault-0.1.0/src/synapse/demo/wiki/open-source.md +13 -0
  37. synapse_vault-0.1.0/src/synapse/demo/wiki/public-speaking.md +15 -0
  38. synapse_vault-0.1.0/src/synapse/demo/wiki/python.md +13 -0
  39. synapse_vault-0.1.0/src/synapse/demo/wiki/riverlight-garden.md +15 -0
  40. synapse_vault-0.1.0/src/synapse/demo/wiki/sqlite.md +15 -0
  41. synapse_vault-0.1.0/src/synapse/graph.py +107 -0
  42. synapse_vault-0.1.0/src/synapse/index.py +119 -0
  43. synapse_vault-0.1.0/src/synapse/ingest.py +77 -0
  44. synapse_vault-0.1.0/src/synapse/llm.py +84 -0
  45. synapse_vault-0.1.0/src/synapse/mcp.py +165 -0
  46. synapse_vault-0.1.0/src/synapse/models.py +22 -0
  47. synapse_vault-0.1.0/src/synapse/query.py +88 -0
  48. synapse_vault-0.1.0/src/synapse/raw.py +102 -0
  49. synapse_vault-0.1.0/src/synapse/server.py +264 -0
  50. synapse_vault-0.1.0/src/synapse/vault.py +65 -0
  51. synapse_vault-0.1.0/src/synapse_vault.egg-info/PKG-INFO +152 -0
  52. synapse_vault-0.1.0/src/synapse_vault.egg-info/SOURCES.txt +63 -0
  53. synapse_vault-0.1.0/src/synapse_vault.egg-info/dependency_links.txt +1 -0
  54. synapse_vault-0.1.0/src/synapse_vault.egg-info/entry_points.txt +2 -0
  55. synapse_vault-0.1.0/src/synapse_vault.egg-info/requires.txt +4 -0
  56. synapse_vault-0.1.0/src/synapse_vault.egg-info/top_level.txt +1 -0
  57. synapse_vault-0.1.0/tests/test_adapters.py +82 -0
  58. synapse_vault-0.1.0/tests/test_build.py +135 -0
  59. synapse_vault-0.1.0/tests/test_config.py +16 -0
  60. synapse_vault-0.1.0/tests/test_graph.py +49 -0
  61. synapse_vault-0.1.0/tests/test_ingest.py +38 -0
  62. synapse_vault-0.1.0/tests/test_mcp.py +67 -0
  63. synapse_vault-0.1.0/tests/test_query.py +44 -0
  64. synapse_vault-0.1.0/tests/test_server.py +52 -0
  65. synapse_vault-0.1.0/tests/test_vault.py +23 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Anshu Yadav
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,152 @@
1
+ Metadata-Version: 2.4
2
+ Name: synapse-vault
3
+ Version: 0.1.0
4
+ Summary: A local-first personal knowledge wiki backed by Markdown and SQLite
5
+ Author: Anshu Yadav
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/anshulyadav1976/synapse
8
+ Project-URL: Documentation, https://github.com/anshulyadav1976/synapse#readme
9
+ Project-URL: Issues, https://github.com/anshulyadav1976/synapse/issues
10
+ Keywords: knowledge-graph,local-first,markdown,mcp,personal-knowledge
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Environment :: Console
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: Operating System :: OS Independent
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Programming Language :: Python :: 3.13
19
+ Classifier: Topic :: Database :: Database Engines/Servers
20
+ Classifier: Topic :: Text Processing :: Indexing
21
+ Requires-Python: >=3.11
22
+ Description-Content-Type: text/markdown
23
+ License-File: LICENSE
24
+ Provides-Extra: dev
25
+ Requires-Dist: pytest>=8; extra == "dev"
26
+ Requires-Dist: ruff>=0.6; extra == "dev"
27
+ Dynamic: license-file
28
+
29
+ <div align="center">
30
+
31
+ # Synapse
32
+
33
+ ![Synapse turns scattered history into a linked local knowledge graph](https://raw.githubusercontent.com/anshulyadav1976/synapse/main/docs/demo.gif)
34
+
35
+ **Point it at text. Get a searchable archive for free. Spend a few cents turning it into a Markdown wiki your agent can query.**
36
+
37
+ [![CI](https://github.com/anshulyadav1976/synapse/actions/workflows/ci.yml/badge.svg)](https://github.com/anshulyadav1976/synapse/actions/workflows/ci.yml)
38
+ [![PyPI](https://img.shields.io/pypi/v/synapse-vault)](https://pypi.org/project/synapse-vault/)
39
+ [![Python 3.11+](https://img.shields.io/badge/python-3.11%2B-3157a4)](https://www.python.org/)
40
+ [![zero runtime dependencies](https://img.shields.io/badge/runtime_dependencies-0-13a36f)](https://github.com/anshulyadav1976/synapse/blob/main/pyproject.toml)
41
+ [![MIT](https://img.shields.io/badge/license-MIT-6f5bd3)](https://github.com/anshulyadav1976/synapse/blob/main/LICENSE)
42
+
43
+ </div>
44
+
45
+ Give Codex a private, local memory without filling every prompt with your history:
46
+
47
+ ```bash
48
+ codex mcp add synapse -- uvx --from synapse-vault synapse mcp --vault /absolute/path/to/my-brain
49
+ ```
50
+
51
+ Nothing from the vault is injected into the agent's prompt. The agent searches first, then reads one relevant Markdown page. A thousand unopened pages cost zero tokens.
52
+
53
+ ## See it in ten seconds
54
+
55
+ ```bash
56
+ uvx --from synapse-vault synapse serve --demo
57
+ ```
58
+
59
+ That opens a populated 20-page graph. It needs no API key, import, database server, or JavaScript build.
60
+
61
+ ## Keep your own history
62
+
63
+ ```bash
64
+ uvx --from synapse-vault synapse init ./my-brain
65
+ uvx --from synapse-vault synapse ingest ~/Downloads/chatgpt-export --vault ./my-brain
66
+ uvx --from synapse-vault synapse serve --vault ./my-brain
67
+ ```
68
+
69
+ Ingest is local and free: it writes Markdown and builds a disposable SQLite FTS5 index. Search works immediately:
70
+
71
+ ```bash
72
+ uvx --from synapse-vault synapse search "the phrase I remember" --vault ./my-brain
73
+ ```
74
+
75
+ To turn raw history into a linked wiki, estimate first and then build a small resumable batch:
76
+
77
+ ```bash
78
+ export SYNAPSE_API_KEY="..."
79
+ uvx --from synapse-vault synapse build --vault ./my-brain --limit 20 --dry-run
80
+ uvx --from synapse-vault synapse build --vault ./my-brain --limit 20
81
+ ```
82
+
83
+ The build pass is optional. It is a plain `for` loop making one OpenAI-compatible chat-completions request per item. Run it against OpenAI or a local Ollama/LM Studio server; stop and resume without paying twice.
84
+
85
+ ## Markdown is the database
86
+
87
+ ```text
88
+ my-brain/
89
+ ├── raw/<source>/<YYYY-MM>/<id>.md immutable imported history
90
+ ├── wiki/<slug>.md linked, editable knowledge pages
91
+ ├── synapse.db disposable FTS5 + graph index
92
+ └── synapse.toml model and owner settings
93
+ ```
94
+
95
+ Delete `synapse.db` and `synapse reindex --vault ./my-brain` recreates it. The durable data is ordinary Markdown that works with git, Obsidian, `grep`, and any editor.
96
+
97
+ ## How it works
98
+
99
+ 1. An adapter streams each source into a tiny `Item` shape with stable IDs and timestamps.
100
+ 2. Ingest writes immutable raw Markdown and indexes it with SQLite FTS5—no model call.
101
+ 3. Build compresses one item and asks any OpenAI-compatible model for complete wiki pages.
102
+ 4. `[[wikilinks]]` become edges; a recursive SQLite CTE handles multi-hop traversal.
103
+ 5. The one-file dashboard, CLI, Python API, REST API, and MCP server all use the same vault.
104
+
105
+ ## Model providers
106
+
107
+ Synapse uses one standard-library HTTP POST to `/chat/completions`; there is no provider SDK.
108
+
109
+ | Provider | Base URL | Status |
110
+ |---|---|---|
111
+ | OpenAI | `https://api.openai.com/v1` | Tested with `gpt-4o-mini` |
112
+ | Ollama | `http://localhost:11434/v1` | Compatible; not yet in CI |
113
+ | LM Studio | `http://localhost:1234/v1` | Compatible; not yet in CI |
114
+ | OpenRouter | `https://openrouter.ai/api/v1` | Compatible; community verification wanted |
115
+ | Groq | `https://api.groq.com/openai/v1` | Compatible; community verification wanted |
116
+ | Together | `https://api.together.xyz/v1` | Compatible; community verification wanted |
117
+ | DeepSeek | `https://api.deepseek.com/v1` | Compatible; community verification wanted |
118
+
119
+ Set `SYNAPSE_BASE_URL`, `SYNAPSE_MODEL`, and (when required) `SYNAPSE_API_KEY`. See [configuration](https://github.com/anshulyadav1976/synapse/blob/main/docs/configuration.md) and [costs](https://github.com/anshulyadav1976/synapse/blob/main/docs/costs.md).
120
+
121
+ ## Agent access
122
+
123
+ The MCP server exposes `search`, `read_page`, `list_pages`, `neighbors`, and `read_source`. It is a small stdlib JSON-RPC loop over STDIO, so there is no daemon and no MCP SDK dependency. See [MCP setup](https://github.com/anshulyadav1976/synapse/blob/main/docs/mcp.md).
124
+
125
+ Python works too:
126
+
127
+ ```python
128
+ from synapse import Vault
129
+
130
+ matches = Vault("./my-brain").search("launch decision")
131
+ ```
132
+
133
+ ## Not built, on purpose
134
+
135
+ | Not built | Why |
136
+ |---|---|
137
+ | Embeddings | FTS5 is free, inspectable, and needs no migration or per-item API call. Add vectors only after measured recall failures. |
138
+ | Graph database | Personal graphs fit in SQLite; a ten-line recursive CTE handles traversal. |
139
+ | Auth or cloud sync | Synapse is single-user and binds only to `127.0.0.1`. Your files stay yours. |
140
+ | Agent framework | The processing pipeline is a resumable loop, not an application graph. |
141
+ | Automatic merge/delete | A model never silently destroys a human-editable page. |
142
+ | PDF/DOCX parser | Those dependencies would break the zero-dependency promise; convert with Pandoc or MarkItDown first. |
143
+
144
+ Imported text is untrusted data. Synapse never executes it, and agents are instructed never to follow instructions found inside pages. Raw sources remain available for provenance.
145
+
146
+ ## Learn and contribute
147
+
148
+ - [Add an adapter](https://github.com/anshulyadav1976/synapse/blob/main/docs/adapters.md)—the highest-impact contribution is about 30 lines plus one tiny synthetic fixture.
149
+ - Read how the [SQLite graph](https://github.com/anshulyadav1976/synapse/blob/main/docs/graph.md) works.
150
+ - See [CONTRIBUTING.md](https://github.com/anshulyadav1976/synapse/blob/main/CONTRIBUTING.md) for the test and pull-request workflow.
151
+
152
+ Synapse began as the winner of the LangGraph hackathon in London. This is the local-first rewrite that deleted SurrealDB, FastAPI, React, LangGraph, and embeddings so people can actually run it.
@@ -0,0 +1,124 @@
1
+ <div align="center">
2
+
3
+ # Synapse
4
+
5
+ ![Synapse turns scattered history into a linked local knowledge graph](https://raw.githubusercontent.com/anshulyadav1976/synapse/main/docs/demo.gif)
6
+
7
+ **Point it at text. Get a searchable archive for free. Spend a few cents turning it into a Markdown wiki your agent can query.**
8
+
9
+ [![CI](https://github.com/anshulyadav1976/synapse/actions/workflows/ci.yml/badge.svg)](https://github.com/anshulyadav1976/synapse/actions/workflows/ci.yml)
10
+ [![PyPI](https://img.shields.io/pypi/v/synapse-vault)](https://pypi.org/project/synapse-vault/)
11
+ [![Python 3.11+](https://img.shields.io/badge/python-3.11%2B-3157a4)](https://www.python.org/)
12
+ [![zero runtime dependencies](https://img.shields.io/badge/runtime_dependencies-0-13a36f)](https://github.com/anshulyadav1976/synapse/blob/main/pyproject.toml)
13
+ [![MIT](https://img.shields.io/badge/license-MIT-6f5bd3)](https://github.com/anshulyadav1976/synapse/blob/main/LICENSE)
14
+
15
+ </div>
16
+
17
+ Give Codex a private, local memory without filling every prompt with your history:
18
+
19
+ ```bash
20
+ codex mcp add synapse -- uvx --from synapse-vault synapse mcp --vault /absolute/path/to/my-brain
21
+ ```
22
+
23
+ Nothing from the vault is injected into the agent's prompt. The agent searches first, then reads one relevant Markdown page. A thousand unopened pages cost zero tokens.
24
+
25
+ ## See it in ten seconds
26
+
27
+ ```bash
28
+ uvx --from synapse-vault synapse serve --demo
29
+ ```
30
+
31
+ That opens a populated 20-page graph. It needs no API key, import, database server, or JavaScript build.
32
+
33
+ ## Keep your own history
34
+
35
+ ```bash
36
+ uvx --from synapse-vault synapse init ./my-brain
37
+ uvx --from synapse-vault synapse ingest ~/Downloads/chatgpt-export --vault ./my-brain
38
+ uvx --from synapse-vault synapse serve --vault ./my-brain
39
+ ```
40
+
41
+ Ingest is local and free: it writes Markdown and builds a disposable SQLite FTS5 index. Search works immediately:
42
+
43
+ ```bash
44
+ uvx --from synapse-vault synapse search "the phrase I remember" --vault ./my-brain
45
+ ```
46
+
47
+ To turn raw history into a linked wiki, estimate first and then build a small resumable batch:
48
+
49
+ ```bash
50
+ export SYNAPSE_API_KEY="..."
51
+ uvx --from synapse-vault synapse build --vault ./my-brain --limit 20 --dry-run
52
+ uvx --from synapse-vault synapse build --vault ./my-brain --limit 20
53
+ ```
54
+
55
+ The build pass is optional. It is a plain `for` loop making one OpenAI-compatible chat-completions request per item. Run it against OpenAI or a local Ollama/LM Studio server; stop and resume without paying twice.
56
+
57
+ ## Markdown is the database
58
+
59
+ ```text
60
+ my-brain/
61
+ ├── raw/<source>/<YYYY-MM>/<id>.md immutable imported history
62
+ ├── wiki/<slug>.md linked, editable knowledge pages
63
+ ├── synapse.db disposable FTS5 + graph index
64
+ └── synapse.toml model and owner settings
65
+ ```
66
+
67
+ Delete `synapse.db` and `synapse reindex --vault ./my-brain` recreates it. The durable data is ordinary Markdown that works with git, Obsidian, `grep`, and any editor.
68
+
69
+ ## How it works
70
+
71
+ 1. An adapter streams each source into a tiny `Item` shape with stable IDs and timestamps.
72
+ 2. Ingest writes immutable raw Markdown and indexes it with SQLite FTS5—no model call.
73
+ 3. Build compresses one item and asks any OpenAI-compatible model for complete wiki pages.
74
+ 4. `[[wikilinks]]` become edges; a recursive SQLite CTE handles multi-hop traversal.
75
+ 5. The one-file dashboard, CLI, Python API, REST API, and MCP server all use the same vault.
76
+
77
+ ## Model providers
78
+
79
+ Synapse uses one standard-library HTTP POST to `/chat/completions`; there is no provider SDK.
80
+
81
+ | Provider | Base URL | Status |
82
+ |---|---|---|
83
+ | OpenAI | `https://api.openai.com/v1` | Tested with `gpt-4o-mini` |
84
+ | Ollama | `http://localhost:11434/v1` | Compatible; not yet in CI |
85
+ | LM Studio | `http://localhost:1234/v1` | Compatible; not yet in CI |
86
+ | OpenRouter | `https://openrouter.ai/api/v1` | Compatible; community verification wanted |
87
+ | Groq | `https://api.groq.com/openai/v1` | Compatible; community verification wanted |
88
+ | Together | `https://api.together.xyz/v1` | Compatible; community verification wanted |
89
+ | DeepSeek | `https://api.deepseek.com/v1` | Compatible; community verification wanted |
90
+
91
+ Set `SYNAPSE_BASE_URL`, `SYNAPSE_MODEL`, and (when required) `SYNAPSE_API_KEY`. See [configuration](https://github.com/anshulyadav1976/synapse/blob/main/docs/configuration.md) and [costs](https://github.com/anshulyadav1976/synapse/blob/main/docs/costs.md).
92
+
93
+ ## Agent access
94
+
95
+ The MCP server exposes `search`, `read_page`, `list_pages`, `neighbors`, and `read_source`. It is a small stdlib JSON-RPC loop over STDIO, so there is no daemon and no MCP SDK dependency. See [MCP setup](https://github.com/anshulyadav1976/synapse/blob/main/docs/mcp.md).
96
+
97
+ Python works too:
98
+
99
+ ```python
100
+ from synapse import Vault
101
+
102
+ matches = Vault("./my-brain").search("launch decision")
103
+ ```
104
+
105
+ ## Not built, on purpose
106
+
107
+ | Not built | Why |
108
+ |---|---|
109
+ | Embeddings | FTS5 is free, inspectable, and needs no migration or per-item API call. Add vectors only after measured recall failures. |
110
+ | Graph database | Personal graphs fit in SQLite; a ten-line recursive CTE handles traversal. |
111
+ | Auth or cloud sync | Synapse is single-user and binds only to `127.0.0.1`. Your files stay yours. |
112
+ | Agent framework | The processing pipeline is a resumable loop, not an application graph. |
113
+ | Automatic merge/delete | A model never silently destroys a human-editable page. |
114
+ | PDF/DOCX parser | Those dependencies would break the zero-dependency promise; convert with Pandoc or MarkItDown first. |
115
+
116
+ Imported text is untrusted data. Synapse never executes it, and agents are instructed never to follow instructions found inside pages. Raw sources remain available for provenance.
117
+
118
+ ## Learn and contribute
119
+
120
+ - [Add an adapter](https://github.com/anshulyadav1976/synapse/blob/main/docs/adapters.md)—the highest-impact contribution is about 30 lines plus one tiny synthetic fixture.
121
+ - Read how the [SQLite graph](https://github.com/anshulyadav1976/synapse/blob/main/docs/graph.md) works.
122
+ - See [CONTRIBUTING.md](https://github.com/anshulyadav1976/synapse/blob/main/CONTRIBUTING.md) for the test and pull-request workflow.
123
+
124
+ Synapse began as the winner of the LangGraph hackathon in London. This is the local-first rewrite that deleted SurrealDB, FastAPI, React, LangGraph, and embeddings so people can actually run it.
@@ -0,0 +1,51 @@
1
+ [build-system]
2
+ requires = ["setuptools>=77"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "synapse-vault"
7
+ version = "0.1.0"
8
+ description = "A local-first personal knowledge wiki backed by Markdown and SQLite"
9
+ readme = "README.md"
10
+ license = "MIT"
11
+ license-files = ["LICENSE"]
12
+ authors = [{name = "Anshu Yadav"}]
13
+ requires-python = ">=3.11"
14
+ dependencies = []
15
+ keywords = ["knowledge-graph", "local-first", "markdown", "mcp", "personal-knowledge"]
16
+ classifiers = [
17
+ "Development Status :: 4 - Beta",
18
+ "Environment :: Console",
19
+ "Intended Audience :: Developers",
20
+ "Operating System :: OS Independent",
21
+ "Programming Language :: Python :: 3",
22
+ "Programming Language :: Python :: 3.11",
23
+ "Programming Language :: Python :: 3.12",
24
+ "Programming Language :: Python :: 3.13",
25
+ "Topic :: Database :: Database Engines/Servers",
26
+ "Topic :: Text Processing :: Indexing",
27
+ ]
28
+
29
+ [project.urls]
30
+ Homepage = "https://github.com/anshulyadav1976/synapse"
31
+ Documentation = "https://github.com/anshulyadav1976/synapse#readme"
32
+ Issues = "https://github.com/anshulyadav1976/synapse/issues"
33
+
34
+ [project.optional-dependencies]
35
+ dev = ["pytest>=8", "ruff>=0.6"]
36
+
37
+ [project.scripts]
38
+ synapse = "synapse.cli:main"
39
+
40
+ [tool.setuptools.packages.find]
41
+ where = ["src"]
42
+
43
+ [tool.setuptools.package-data]
44
+ synapse = ["dashboard.html", "demo/wiki/*.md", "demo/raw/demo/*/*.md"]
45
+
46
+ [tool.pytest.ini_options]
47
+ pythonpath = ["src"]
48
+
49
+ [tool.ruff]
50
+ line-length = 100
51
+ target-version = "py311"
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,7 @@
1
+ """Synapse's public Python API."""
2
+
3
+ from .vault import Vault
4
+
5
+ __all__ = ["Vault"]
6
+ __version__ = "0.1.0"
7
+
@@ -0,0 +1,4 @@
1
+ from .cli import main
2
+
3
+ raise SystemExit(main())
4
+
@@ -0,0 +1,70 @@
1
+ """Small reader registry: every input format stops at Item."""
2
+
3
+ from collections.abc import Callable, Iterator
4
+ from pathlib import Path
5
+ from typing import TypeAlias
6
+ from zipfile import BadZipFile, ZipFile
7
+
8
+ from ..models import Item
9
+
10
+ Reader: TypeAlias = Callable[[Path], Iterator[Item]]
11
+ READERS: dict[str, Reader] = {}
12
+
13
+
14
+ def adapter(name: str) -> Callable[[Reader], Reader]:
15
+ def register(reader: Reader) -> Reader:
16
+ READERS[name] = reader
17
+ return reader
18
+
19
+ return register
20
+
21
+
22
+ def _zip_names(path: Path) -> list[str]:
23
+ try:
24
+ with ZipFile(path) as archive:
25
+ return archive.namelist()
26
+ except (BadZipFile, OSError):
27
+ return []
28
+
29
+
30
+ def detect(path: Path) -> str:
31
+ if path.is_dir():
32
+ names = {child.name for child in path.iterdir()}
33
+ if "export_manifest.json" in names or any(
34
+ name.startswith("conversations-") and name.endswith(".json") for name in names
35
+ ):
36
+ return "chatgpt"
37
+ return "dir"
38
+ if path.name == "conversations.json" or (
39
+ path.name.startswith("conversations-") and path.suffix == ".json"
40
+ ):
41
+ return "chatgpt"
42
+ if path.name == "chat.html":
43
+ return "chatgpt"
44
+ if path.suffix.lower() == ".zip":
45
+ names = _zip_names(path)
46
+ if any(
47
+ Path(name).name == "export_manifest.json"
48
+ or Path(name).name == "conversations.json"
49
+ or Path(name).name == "chat.html"
50
+ or (Path(name).name.startswith("conversations-") and name.endswith(".json"))
51
+ for name in names
52
+ ):
53
+ return "chatgpt"
54
+ return "file"
55
+
56
+
57
+ def read(path: str | Path, format_name: str | None = None) -> Iterator[Item]:
58
+ source = Path(path).expanduser().resolve()
59
+ name = format_name or detect(source)
60
+ try:
61
+ reader = READERS[name]
62
+ except KeyError as error:
63
+ choices = ", ".join(sorted(READERS))
64
+ raise ValueError(f"Unknown format {name!r}. Available formats: {choices}") from error
65
+ yield from reader(source)
66
+
67
+
68
+ # Importing registers the built-in readers while keeping contributor adapters tiny.
69
+ from . import chatgpt, files # noqa: F401
70
+
@@ -0,0 +1,152 @@
1
+ """Reader for modern and legacy ChatGPT exports."""
2
+
3
+ import json
4
+ import mmap
5
+ import re
6
+ from collections.abc import Iterable, Iterator
7
+ from datetime import UTC, datetime
8
+ from pathlib import Path, PurePosixPath
9
+ from typing import Any
10
+ from zipfile import ZipFile
11
+
12
+ from ..models import Item, Turn
13
+ from . import adapter
14
+
15
+ SHARD = re.compile(r"conversations-\d+\.json$")
16
+
17
+
18
+ def _iso(value: object) -> str:
19
+ try:
20
+ return datetime.fromtimestamp(float(value), UTC).isoformat()
21
+ except (TypeError, ValueError, OSError):
22
+ return datetime.fromtimestamp(0, UTC).isoformat()
23
+
24
+
25
+ def _parts(content: dict[str, Any]) -> str:
26
+ return "\n".join(part for part in content.get("parts", []) if isinstance(part, str)).strip()
27
+
28
+
29
+ def _conversation(data: dict[str, Any]) -> Item:
30
+ messages: list[tuple[float, int, Turn]] = []
31
+ for order, node in enumerate(data.get("mapping", {}).values()):
32
+ message = node.get("message") or {}
33
+ text = _parts(message.get("content") or {})
34
+ if not text:
35
+ continue
36
+ author = message.get("author") or {}
37
+ role = str(author.get("role") or "unknown")
38
+ speaker = "me" if role == "user" else str(author.get("name") or role)
39
+ created = message.get("create_time")
40
+ messages.append((float(created or 0), order, Turn(speaker, text, _iso(created))))
41
+ messages.sort(key=lambda entry: (entry[0], entry[1]))
42
+ identifier = str(data.get("conversation_id") or data.get("id") or "")
43
+ if not identifier:
44
+ raise ValueError("A ChatGPT conversation is missing both conversation_id and id")
45
+ return Item(
46
+ id=identifier,
47
+ source="chatgpt",
48
+ title=str(data.get("title") or "Untitled conversation"),
49
+ ts=_iso(data.get("create_time")),
50
+ turns=[entry[2] for entry in messages],
51
+ )
52
+
53
+
54
+ def _items(conversations: Iterable[dict[str, Any]]) -> Iterator[Item]:
55
+ for conversation in conversations:
56
+ yield _conversation(conversation)
57
+
58
+
59
+ def _manifest_files(data: dict[str, Any]) -> list[str]:
60
+ return [
61
+ str(entry["path"])
62
+ for entry in data.get("export_files", [])
63
+ if isinstance(entry, dict)
64
+ and "path" in entry
65
+ and (SHARD.search(str(entry["path"])) or PurePosixPath(str(entry["path"])).name == "conversations.json")
66
+ ]
67
+
68
+
69
+ def _folder_files(path: Path) -> list[Path]:
70
+ manifest = path / "export_manifest.json"
71
+ if manifest.exists():
72
+ names = _manifest_files(json.loads(manifest.read_text(encoding="utf-8")))
73
+ found = [path / name for name in names if (path / name).is_file()]
74
+ if found:
75
+ return found
76
+ shards = sorted(path.glob("conversations-*.json"))
77
+ if shards:
78
+ return shards
79
+ legacy = path / "conversations.json"
80
+ return [legacy] if legacy.exists() else []
81
+
82
+
83
+ def _from_html_bytes(data: bytes, label: str) -> Iterator[Item]:
84
+ marker = b"var jsonData ="
85
+ start = data.find(marker)
86
+ if start < 0:
87
+ raise ValueError(f"No 'var jsonData =' conversation payload found in {label}")
88
+ start += len(marker)
89
+ end = data.find(b";</script>", start)
90
+ if end < 0:
91
+ end = data.find(b";\n", start)
92
+ if end < 0:
93
+ raise ValueError(f"The ChatGPT payload in {label} has no closing semicolon")
94
+ yield from _items(json.loads(data[start:end]))
95
+
96
+
97
+ def _from_html(path: Path) -> Iterator[Item]:
98
+ with path.open("rb") as handle, mmap.mmap(handle.fileno(), 0, access=mmap.ACCESS_READ) as data:
99
+ yield from _from_html_bytes(data, str(path))
100
+
101
+
102
+ def _from_zip(path: Path) -> Iterator[Item]:
103
+ with ZipFile(path) as archive:
104
+ names = archive.namelist()
105
+ manifest_name = next((name for name in names if PurePosixPath(name).name == "export_manifest.json"), None)
106
+ selected: list[str] = []
107
+ if manifest_name:
108
+ manifest = json.loads(archive.read(manifest_name))
109
+ base = PurePosixPath(manifest_name).parent
110
+ available = set(names)
111
+ selected = [str(base / name) for name in _manifest_files(manifest) if str(base / name) in available]
112
+ if not selected:
113
+ selected = sorted(
114
+ name
115
+ for name in names
116
+ if SHARD.search(PurePosixPath(name).name)
117
+ or PurePosixPath(name).name == "conversations.json"
118
+ )
119
+ if selected:
120
+ for name in selected:
121
+ yield from _items(json.loads(archive.read(name)))
122
+ return
123
+ html = next((name for name in names if PurePosixPath(name).name == "chat.html"), None)
124
+ if html:
125
+ yield from _from_html_bytes(archive.read(html), f"{path}!{html}")
126
+ return
127
+ raise ValueError(f"No ChatGPT conversations found in {path}")
128
+
129
+
130
+ @adapter("chatgpt")
131
+ def read_chatgpt(path: Path) -> Iterator[Item]:
132
+ if path.is_dir():
133
+ files = _folder_files(path)
134
+ if not files:
135
+ html = path / "chat.html"
136
+ if html.exists():
137
+ yield from _from_html(html)
138
+ return
139
+ raise ValueError(
140
+ f"No conversations found in {path}. Modern exports shard them as "
141
+ "conversations-000.json; pass --format chatgpt to force detection."
142
+ )
143
+ for file in files:
144
+ yield from _items(json.loads(file.read_text(encoding="utf-8")))
145
+ return
146
+ if path.suffix.lower() == ".zip":
147
+ yield from _from_zip(path)
148
+ elif path.name == "chat.html":
149
+ yield from _from_html(path)
150
+ else:
151
+ yield from _items(json.loads(path.read_text(encoding="utf-8")))
152
+