memtrails 0.4.1__tar.gz

memtrails-0.4.1/.gitignore

@@ -0,0 +1,8 @@
+.venv/
+__pycache__/
+*.egg-info/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+dist/
+build/

memtrails-0.4.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Corneliu Maftuleac
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

memtrails-0.4.1/PKG-INFO

@@ -0,0 +1,237 @@
+Metadata-Version: 2.5
+Name: memtrails
+Version: 0.4.1
+Summary: Retrieval and exploration for any text — e-mail, chat, notes, documents, news, commits, conversations. Searchable at once, on your own machine, in any language; optional consolidation into typed knowledge with a language model.
+Project-URL: Homepage, https://github.com/cmaftuleac/memtrails
+Project-URL: Repository, https://github.com/cmaftuleac/memtrails
+Project-URL: Issues, https://github.com/cmaftuleac/memtrails/issues
+Author: Corneliu Maftuleac
+License-Expression: MIT
+License-File: LICENSE
+Keywords: agent-memory,colbert,embeddings,knowledge-graph,late-interaction,llm,long-term-memory,maxsim,mcp,memory,multilingual,qdrant,rag,retrieval,search
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: MacOS
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Text Processing :: Indexing
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.27
+Requires-Dist: icu4py>=1.1
+Requires-Dist: numpy>=1.24
+Requires-Dist: onnxruntime==1.23.2
+Requires-Dist: regex>=2024.4.16
+Requires-Dist: tokenizers>=0.20
+Provides-Extra: dev
+Requires-Dist: mypy==2.4.1; extra == 'dev'
+Requires-Dist: pytest-asyncio==1.4.1.post1; extra == 'dev'
+Requires-Dist: pytest==9.2.0; extra == 'dev'
+Requires-Dist: ruff==0.17.3; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# memtrails for Python
+
+**Retrieval and exploration for any text — e-mail, chat, notes, memos, documents, news, commit messages, conversations.**
+
+A native Python port of [memtrails](https://github.com/cmaftuleac/memtrails) (npm `memtrails` 0.4.1): the same algorithms,
+constants, prompts, JSON Schemas and files, written in Python and natively async. Everything you add is searchable the
+moment it is added, on your own machine, with no server and no language model: a local embedding model, a keyword
+channel, and a word-by-word reordering that needs no reranker. Give it a language model too, and it consolidates what it
+holds into typed knowledge, dated events and profiles — and an agent can explore it.
+
+```python
+import asyncio
+from memtrails import open_memory
+
+async def main() -> None:
+    async with await open_memory(home="./memory") as memory:
+        await memory.add([
+            {"id": "msg-1", "kind": "email", "author": "Ana", "title": "Proxy certificate expired", "at": "2026-04-02",
+             "text": "The TLS certificate expired on the 1st. Renewal stopped because the cron job was removed in March."},
+            {"id": "c9f1e2", "kind": "commit", "author": "Ion", "title": "Restore certificate renewal cron", "at": "2026-04-03",
+             "text": "Files: deploy/cron/renew.sh"},
+        ])
+        hits = await memory.search({"query": "why did the certificate stop renewing?"})
+        # [{"id": "stm:…", "kind": "email", "title": …, "summary": …, "ts": …, "record": {"id": "msg-1", "author": "Ana"}, "score": …}, …]
+
+asyncio.run(main())
+```
+
+## Install
+
+```bash
+pip install memtrails
+```
+
+Python 3.10 to 3.13, on macOS or Linux (x64 or arm64). The first `open_memory` downloads the embedding model (218 MB, from
+Hugging Face) and the Qdrant binary (~30 MB, from Qdrant's GitHub release) into `~/.cache/memtrails`, each checked against
+a pinned SHA-256 — the same files, in the same folder, as the TypeScript library, so the two share one download. Set
+`MEMTRAILS_CACHE` to put them elsewhere.
+
+## One memory, two languages
+
+A memory folder written by the TypeScript library opens and searches in Python with the same results, and the other way
+round: the same `records.jsonl` and `knowledge.jsonl` logs (crash recovery included), `words.bin`, `usage.json`,
+`recall.jsonl`, `index.json`, `memory.lock`, `rebuild.json` and `sleep.json`; the same Qdrant collection (named vectors
+`dense` and `bm25`, payload fields, point ids, index layout 7); the same record ids and fingerprints; the same keyword
+tokens (ICU word segmentation, through `icu4py`, which ships ICU 78 — the reference's); and, on the same ONNX Runtime
+(1.23.2, pinned) and the same machine, the very same vectors (its 4-bit kernels differ slightly between CPU
+architectures, in both libraries). The one-process-a-home lock is shared: a memory open in Node cannot be opened in
+Python at the same time, and the other way round.
+
+Strings are measured as JavaScript measures them (UTF-16 code units), JSON is written as `JSON.stringify` writes it, and
+dates are read as V8's `Date.parse` reads them, local time and mail dates included. Every language-model request — system
+text, prompt and JSON Schema — is the reference's, byte for byte.
+
+The port is held to the conformance fixtures in [`../conformance`](../conformance): what the TypeScript library gives for
+fixed inputs (text, words, ids, dates, records, the secret filter, search text, importers, prompts, fused search, vectors).
+
+## Adding data
+
+A record is anything with text. Only `id` and `text` are required.
+
+```python
+await memory.add({
+    "id": "<CAF8w@mail.example>",   # unique within its source; adding the same id again updates it
+    "source": "work-mailbox",       # part of the identity
+    "kind": "email",                # any word: email, message, note, memo, article, commit, ticket, …
+    "collection": "ops",            # search can be scoped to collections
+    "thread": "proxy-cert",         # records that belong together
+    "author": "Ana Popescu",
+    "title": "Proxy certificate expired",
+    "text": "…",
+    "at": "2026-04-02T09:14:00Z",   # ms, an ISO or RFC 2822 string, or a datetime
+    "url": "https://mail.example/…",
+    "metadata": {"to": "ops@example.com"},
+})
+
+await memory.add_conversation({"id": "chat-42", "turns": [
+    {"role": "user", "text": "Where do we have lunch on Friday?"},
+    {"role": "assistant", "text": "At the Italian place near the office, at one."},
+]})
+```
+
+### Importers
+
+Each importer is an async generator: records stream one at a time, never the whole source in memory.
+
+```python
+from memtrails.importers import from_mbox, from_eml, from_slack_export, from_git, from_files, from_jsonl, batches
+
+await memory.import_(from_mbox("./archive.mbox"))                  # e-mail (mbox / .eml)
+await memory.import_(from_slack_export("./slack-export"))          # a Slack workspace export
+await memory.import_(from_git("./my-repo", since="2025-01-01"))    # commit messages and changed files
+await memory.import_(from_files("./notes"))                        # Markdown, text and HTML files
+await memory.import_(from_jsonl("./records.jsonl"))                # one record per line
+
+async for record in from_mbox("./archive.mbox"):
+    ...
+```
+
+`import_` takes any async iterable of records (a plain iterable too), so a database cursor or an API client works the
+same way. Unchanged records cost nothing; `await memory.remove([...ids])` forgets.
+
+## Searching
+
+```python
+await memory.search({"query": "who renewed the certificate?", "limit": 8})
+await memory.search({"query": "standup notes", "collections": ["ops"], "kinds": ["message"], "from": "2026-04", "to": "2026-04"})
+await memory.search_text({"query": "what happened with the proxy?", "order": "time"})   # as a reader is shown it
+await memory.room({"query": "what made the renewal stop?", "chars": 6000})             # the best sentences
+await memory.open(hits[0]["id"])                                                         # one item whole
+await memory.explore({"topic": "the proxy"})                                             # what the concept graph reaches
+```
+
+Results are plain dicts keyed as the reference keys them (`eventAt`, `startAt`, `endAt`, `match`, `record`), so they
+serialise as they are and read the same in either language.
+
+## Consolidation and sleep (optional, with a language model)
+
+```python
+memory = await open_memory(
+    home="./memory",
+    llm={"model": "gpt-6-luna", "reasoning_effort": "low", "api_key": os.environ["OPENAI_API_KEY"]},  # or base_url for any OpenAI-compatible server
+)
+await memory.consolidate()   # after adding
+await memory.sleep()         # once a day
+await memory.gather({"query": "every plumber we called this year"}, notes=True)
+```
+
+Any model works: pass `llm={"base_url": …, "model": …, "api_key": …}` for an OpenAI-compatible server, or your own async
+function:
+
+```python
+from memtrails import LlmRequest
+
+async def ask(req: LlmRequest) -> dict:
+    ...   # req.system, req.prompt, req.schema (strict JSON Schema), req.name, req.max_tokens, req.timeout_ms
+    return {"answer": {...}}   # or {"unavailable": "why"}
+```
+
+Secrets (keys, tokens, credentials in URLs, private keys) are scrubbed from everything sent to a language model and from
+what consolidation keeps, exactly as the reference scrubs them. `OPENAI_API_KEY` is sent only to OpenAI's own address.
+
+## For agents
+
+```python
+for tool in memory.tools():          # memory_search, memory_open, memory_explore
+    tool.name, tool.description, tool.json_schema
+    result = await tool.handler({"query": "proxy certificate"})   # {"content": [{"type": "text", "text": …}]}
+```
+
+The schemas are the reference's, key for key; the handler checks its arguments itself (texts up to 2,000 characters, up
+to 50 collections) and answers a bad call with zod's words.
+
+## Configuration
+
+| Option of `open_memory` | Default | |
+|---|---|---|
+| `home` | — | Where this memory lives: its logs, index and word vectors (made private, 0700). |
+| `embedding` | the model fetched into the cache | `{"model_dir": …}`, `{"server": {"url", "api_key"?, "model"?, "gzip"?}}`, `{"threads": 2}`. |
+| `rerank` | `"words"` | `"words"`, a rerank server `{"url", …}`, your own async function, or `False`. |
+| `llm` | from the environment | Your own async `Ask`, or `{"model", "api_key"?, "base_url"?, "reasoning_effort"?, "on_usage"?, "retries"?}`. |
+| `qdrant` | a process of its own | `{"url", "api_key"?, "collection"?}` for your Qdrant server, or `{"binary": …}`. |
+| `concurrency` | 4 | Windows the language model reads at once. |
+| `http` | one of its own | The `httpx.AsyncClient` every call goes through (yours stays open when the memory closes). |
+| `log` | silent | A function of one line. |
+
+The environment variables are the reference's: `MEMTRAILS_CACHE`, `MEMTRAILS_MODEL`, `MEMTRAILS_LLM_API_KEY` or
+`OPENAI_API_KEY`, `MEMTRAILS_LLM_BASE_URL`, `MEMTRAILS_EMBED_URL` / `_KEY` / `_MODEL`, `MEMTRAILS_RERANK_URL` / `_KEY` / `_MODEL`.
+
+## Async all the way down
+
+Every method that does I/O is `async def`, and a memory is an async context manager. A memory belongs to the event loop
+that opened it: use it, and close it, from that loop. One `httpx.AsyncClient` reaches a Qdrant of yours, the embedding
+and rerank servers, the language model and the downloads (streamed, given up after 60 s without a byte); memory's own
+Qdrant is reached through a client of its own that takes no proxy from the environment. Qdrant, `tar` and `git` run as
+asyncio subprocesses; Qdrant is supervised and restarted by an asyncio task, bound to 127.0.0.1 with a random key,
+proven to be ours before it gets the key, and given a minimal environment. The model's inference (one at a time, under
+an `asyncio.Lock`), word-by-word scoring, and the reading or rewriting of whole files run on threads, so the event loop
+never waits on them. Consolidate, sleep, rebuild, index_words and remove take turns under an `asyncio.Lock`, and an add
+waits while a rebuild runs; `close()` stops the running one at its next step and waits for it and for the adds and
+searches in flight.
+
+## Known differences
+
+- **Word-like segments**: ICU tells JavaScript whether a segment is a word (its rule status); `icu4py` gives the segments
+  but not the status, so it is read off the segment. On adversarial mixtures of every script it agrees with Node except
+  for a combining mark left alone after a Hangul syllable or a `‿` joiner — text no one writes. Boundaries agree everywhere.
+- **Unicode versions**: `fold` uses Python's `unicodedata` (Unicode 13 on Python 3.10, 15.1 on 3.13) where Node 24 has
+  Unicode 17: characters added since fold differently or not at all.
+- **Collation**: `localeCompare` breaks two ties (a profile's name among equally common spellings, items of the same time);
+  Python has no ICU collator here, so the root collation is approximated (punctuation, digits, letters by base, then
+  accents, then case). It agrees on every name memory compares in the tests.
+- **ONNX Runtime**: pinned to 1.23.2 for identical vectors; Python 3.14 waits for a runtime that both libraries pin.
+- **Windows** has the pieces it needs but is not tested yet (as with the TypeScript library).
+
+## Licence
+
+MIT © Corneliu Maftuleac. Third parties: EmbeddingGemma is Google's, under the
+[Gemma Terms of Use](https://ai.google.dev/gemma/terms), downloaded from Hugging Face, not shipped; Qdrant is Apache-2.0,
+downloaded from its release; ICU (in `icu4py`) is under the Unicode licence.

memtrails-0.4.1/README.md

@@ -0,0 +1,200 @@
+# memtrails for Python
+
+**Retrieval and exploration for any text — e-mail, chat, notes, memos, documents, news, commit messages, conversations.**
+
+A native Python port of [memtrails](https://github.com/cmaftuleac/memtrails) (npm `memtrails` 0.4.1): the same algorithms,
+constants, prompts, JSON Schemas and files, written in Python and natively async. Everything you add is searchable the
+moment it is added, on your own machine, with no server and no language model: a local embedding model, a keyword
+channel, and a word-by-word reordering that needs no reranker. Give it a language model too, and it consolidates what it
+holds into typed knowledge, dated events and profiles — and an agent can explore it.
+
+```python
+import asyncio
+from memtrails import open_memory
+
+async def main() -> None:
+    async with await open_memory(home="./memory") as memory:
+        await memory.add([
+            {"id": "msg-1", "kind": "email", "author": "Ana", "title": "Proxy certificate expired", "at": "2026-04-02",
+             "text": "The TLS certificate expired on the 1st. Renewal stopped because the cron job was removed in March."},
+            {"id": "c9f1e2", "kind": "commit", "author": "Ion", "title": "Restore certificate renewal cron", "at": "2026-04-03",
+             "text": "Files: deploy/cron/renew.sh"},
+        ])
+        hits = await memory.search({"query": "why did the certificate stop renewing?"})
+        # [{"id": "stm:…", "kind": "email", "title": …, "summary": …, "ts": …, "record": {"id": "msg-1", "author": "Ana"}, "score": …}, …]
+
+asyncio.run(main())
+```
+
+## Install
+
+```bash
+pip install memtrails
+```
+
+Python 3.10 to 3.13, on macOS or Linux (x64 or arm64). The first `open_memory` downloads the embedding model (218 MB, from
+Hugging Face) and the Qdrant binary (~30 MB, from Qdrant's GitHub release) into `~/.cache/memtrails`, each checked against
+a pinned SHA-256 — the same files, in the same folder, as the TypeScript library, so the two share one download. Set
+`MEMTRAILS_CACHE` to put them elsewhere.
+
+## One memory, two languages
+
+A memory folder written by the TypeScript library opens and searches in Python with the same results, and the other way
+round: the same `records.jsonl` and `knowledge.jsonl` logs (crash recovery included), `words.bin`, `usage.json`,
+`recall.jsonl`, `index.json`, `memory.lock`, `rebuild.json` and `sleep.json`; the same Qdrant collection (named vectors
+`dense` and `bm25`, payload fields, point ids, index layout 7); the same record ids and fingerprints; the same keyword
+tokens (ICU word segmentation, through `icu4py`, which ships ICU 78 — the reference's); and, on the same ONNX Runtime
+(1.23.2, pinned) and the same machine, the very same vectors (its 4-bit kernels differ slightly between CPU
+architectures, in both libraries). The one-process-a-home lock is shared: a memory open in Node cannot be opened in
+Python at the same time, and the other way round.
+
+Strings are measured as JavaScript measures them (UTF-16 code units), JSON is written as `JSON.stringify` writes it, and
+dates are read as V8's `Date.parse` reads them, local time and mail dates included. Every language-model request — system
+text, prompt and JSON Schema — is the reference's, byte for byte.
+
+The port is held to the conformance fixtures in [`../conformance`](../conformance): what the TypeScript library gives for
+fixed inputs (text, words, ids, dates, records, the secret filter, search text, importers, prompts, fused search, vectors).
+
+## Adding data
+
+A record is anything with text. Only `id` and `text` are required.
+
+```python
+await memory.add({
+    "id": "<CAF8w@mail.example>",   # unique within its source; adding the same id again updates it
+    "source": "work-mailbox",       # part of the identity
+    "kind": "email",                # any word: email, message, note, memo, article, commit, ticket, …
+    "collection": "ops",            # search can be scoped to collections
+    "thread": "proxy-cert",         # records that belong together
+    "author": "Ana Popescu",
+    "title": "Proxy certificate expired",
+    "text": "…",
+    "at": "2026-04-02T09:14:00Z",   # ms, an ISO or RFC 2822 string, or a datetime
+    "url": "https://mail.example/…",
+    "metadata": {"to": "ops@example.com"},
+})
+
+await memory.add_conversation({"id": "chat-42", "turns": [
+    {"role": "user", "text": "Where do we have lunch on Friday?"},
+    {"role": "assistant", "text": "At the Italian place near the office, at one."},
+]})
+```
+
+### Importers
+
+Each importer is an async generator: records stream one at a time, never the whole source in memory.
+
+```python
+from memtrails.importers import from_mbox, from_eml, from_slack_export, from_git, from_files, from_jsonl, batches
+
+await memory.import_(from_mbox("./archive.mbox"))                  # e-mail (mbox / .eml)
+await memory.import_(from_slack_export("./slack-export"))          # a Slack workspace export
+await memory.import_(from_git("./my-repo", since="2025-01-01"))    # commit messages and changed files
+await memory.import_(from_files("./notes"))                        # Markdown, text and HTML files
+await memory.import_(from_jsonl("./records.jsonl"))                # one record per line
+
+async for record in from_mbox("./archive.mbox"):
+    ...
+```
+
+`import_` takes any async iterable of records (a plain iterable too), so a database cursor or an API client works the
+same way. Unchanged records cost nothing; `await memory.remove([...ids])` forgets.
+
+## Searching
+
+```python
+await memory.search({"query": "who renewed the certificate?", "limit": 8})
+await memory.search({"query": "standup notes", "collections": ["ops"], "kinds": ["message"], "from": "2026-04", "to": "2026-04"})
+await memory.search_text({"query": "what happened with the proxy?", "order": "time"})   # as a reader is shown it
+await memory.room({"query": "what made the renewal stop?", "chars": 6000})             # the best sentences
+await memory.open(hits[0]["id"])                                                         # one item whole
+await memory.explore({"topic": "the proxy"})                                             # what the concept graph reaches
+```
+
+Results are plain dicts keyed as the reference keys them (`eventAt`, `startAt`, `endAt`, `match`, `record`), so they
+serialise as they are and read the same in either language.
+
+## Consolidation and sleep (optional, with a language model)
+
+```python
+memory = await open_memory(
+    home="./memory",
+    llm={"model": "gpt-6-luna", "reasoning_effort": "low", "api_key": os.environ["OPENAI_API_KEY"]},  # or base_url for any OpenAI-compatible server
+)
+await memory.consolidate()   # after adding
+await memory.sleep()         # once a day
+await memory.gather({"query": "every plumber we called this year"}, notes=True)
+```
+
+Any model works: pass `llm={"base_url": …, "model": …, "api_key": …}` for an OpenAI-compatible server, or your own async
+function:
+
+```python
+from memtrails import LlmRequest
+
+async def ask(req: LlmRequest) -> dict:
+    ...   # req.system, req.prompt, req.schema (strict JSON Schema), req.name, req.max_tokens, req.timeout_ms
+    return {"answer": {...}}   # or {"unavailable": "why"}
+```
+
+Secrets (keys, tokens, credentials in URLs, private keys) are scrubbed from everything sent to a language model and from
+what consolidation keeps, exactly as the reference scrubs them. `OPENAI_API_KEY` is sent only to OpenAI's own address.
+
+## For agents
+
+```python
+for tool in memory.tools():          # memory_search, memory_open, memory_explore
+    tool.name, tool.description, tool.json_schema
+    result = await tool.handler({"query": "proxy certificate"})   # {"content": [{"type": "text", "text": …}]}
+```
+
+The schemas are the reference's, key for key; the handler checks its arguments itself (texts up to 2,000 characters, up
+to 50 collections) and answers a bad call with zod's words.
+
+## Configuration
+
+| Option of `open_memory` | Default | |
+|---|---|---|
+| `home` | — | Where this memory lives: its logs, index and word vectors (made private, 0700). |
+| `embedding` | the model fetched into the cache | `{"model_dir": …}`, `{"server": {"url", "api_key"?, "model"?, "gzip"?}}`, `{"threads": 2}`. |
+| `rerank` | `"words"` | `"words"`, a rerank server `{"url", …}`, your own async function, or `False`. |
+| `llm` | from the environment | Your own async `Ask`, or `{"model", "api_key"?, "base_url"?, "reasoning_effort"?, "on_usage"?, "retries"?}`. |
+| `qdrant` | a process of its own | `{"url", "api_key"?, "collection"?}` for your Qdrant server, or `{"binary": …}`. |
+| `concurrency` | 4 | Windows the language model reads at once. |
+| `http` | one of its own | The `httpx.AsyncClient` every call goes through (yours stays open when the memory closes). |
+| `log` | silent | A function of one line. |
+
+The environment variables are the reference's: `MEMTRAILS_CACHE`, `MEMTRAILS_MODEL`, `MEMTRAILS_LLM_API_KEY` or
+`OPENAI_API_KEY`, `MEMTRAILS_LLM_BASE_URL`, `MEMTRAILS_EMBED_URL` / `_KEY` / `_MODEL`, `MEMTRAILS_RERANK_URL` / `_KEY` / `_MODEL`.
+
+## Async all the way down
+
+Every method that does I/O is `async def`, and a memory is an async context manager. A memory belongs to the event loop
+that opened it: use it, and close it, from that loop. One `httpx.AsyncClient` reaches a Qdrant of yours, the embedding
+and rerank servers, the language model and the downloads (streamed, given up after 60 s without a byte); memory's own
+Qdrant is reached through a client of its own that takes no proxy from the environment. Qdrant, `tar` and `git` run as
+asyncio subprocesses; Qdrant is supervised and restarted by an asyncio task, bound to 127.0.0.1 with a random key,
+proven to be ours before it gets the key, and given a minimal environment. The model's inference (one at a time, under
+an `asyncio.Lock`), word-by-word scoring, and the reading or rewriting of whole files run on threads, so the event loop
+never waits on them. Consolidate, sleep, rebuild, index_words and remove take turns under an `asyncio.Lock`, and an add
+waits while a rebuild runs; `close()` stops the running one at its next step and waits for it and for the adds and
+searches in flight.
+
+## Known differences
+
+- **Word-like segments**: ICU tells JavaScript whether a segment is a word (its rule status); `icu4py` gives the segments
+  but not the status, so it is read off the segment. On adversarial mixtures of every script it agrees with Node except
+  for a combining mark left alone after a Hangul syllable or a `‿` joiner — text no one writes. Boundaries agree everywhere.
+- **Unicode versions**: `fold` uses Python's `unicodedata` (Unicode 13 on Python 3.10, 15.1 on 3.13) where Node 24 has
+  Unicode 17: characters added since fold differently or not at all.
+- **Collation**: `localeCompare` breaks two ties (a profile's name among equally common spellings, items of the same time);
+  Python has no ICU collator here, so the root collation is approximated (punctuation, digits, letters by base, then
+  accents, then case). It agrees on every name memory compares in the tests.
+- **ONNX Runtime**: pinned to 1.23.2 for identical vectors; Python 3.14 waits for a runtime that both libraries pin.
+- **Windows** has the pieces it needs but is not tested yet (as with the TypeScript library).
+
+## Licence
+
+MIT © Corneliu Maftuleac. Third parties: EmbeddingGemma is Google's, under the
+[Gemma Terms of Use](https://ai.google.dev/gemma/terms), downloaded from Hugging Face, not shipped; Qdrant is Apache-2.0,
+downloaded from its release; ICU (in `icu4py`) is under the Unicode licence.

memtrails-0.4.1/pyproject.toml

@@ -0,0 +1,78 @@
+[build-system]
+requires = ["hatchling>=1.25"]
+build-backend = "hatchling.build"
+
+[project]
+name = "memtrails"
+version = "0.4.1"
+description = "Retrieval and exploration for any text — e-mail, chat, notes, documents, news, commits, conversations. Searchable at once, on your own machine, in any language; optional consolidation into typed knowledge with a language model."
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Corneliu Maftuleac" }]
+requires-python = ">=3.10"
+keywords = ["retrieval", "search", "memory", "rag", "agent-memory", "long-term-memory", "embeddings", "late-interaction",
+  "colbert", "maxsim", "qdrant", "knowledge-graph", "llm", "mcp", "multilingual"]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Intended Audience :: Developers",
+  "Operating System :: MacOS",
+  "Operating System :: POSIX :: Linux",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Topic :: Scientific/Engineering :: Artificial Intelligence",
+  "Topic :: Text Processing :: Indexing",
+  "Typing :: Typed",
+]
+dependencies = [
+  "httpx>=0.27",
+  "icu4py>=1.1",
+  "numpy>=1.24",
+  "onnxruntime==1.23.2",
+  "regex>=2024.4.16",
+  "tokenizers>=0.20",
+]
+
+[project.optional-dependencies]
+dev = ["mypy==2.4.1", "pytest==9.2.0", "pytest-asyncio==1.4.1.post1", "ruff==0.17.3"]
+
+[project.urls]
+Homepage = "https://github.com/cmaftuleac/memtrails"
+Repository = "https://github.com/cmaftuleac/memtrails"
+Issues = "https://github.com/cmaftuleac/memtrails/issues"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/memtrails"]
+
+[tool.hatch.build.targets.sdist]
+include = ["src/memtrails", "README.md", "LICENSE", "pyproject.toml"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "function"
+markers = ["integration: needs the real embedding model and a real Qdrant (MEMTRAILS_INTEGRATION=1)"]
+filterwarnings = ["error::pytest.PytestUnraisableExceptionWarning"]
+
+[tool.ruff]
+line-length = 140
+target-version = "py310"
+src = ["src", "tests"]
+
+[tool.ruff.lint]
+select = ["E", "F", "W", "I", "B", "UP", "SIM", "RUF", "BLE"]
+ignore = ["RUF001", "RUF002", "RUF003", "SIM108", "B008", "SIM102", "SIM105", "SIM401", "SIM112", "E501"]
+
+[tool.mypy]
+python_version = "3.10"
+strict = true
+warn_return_any = false
+disallow_any_generics = false
+files = ["src/memtrails"]
+
+[[tool.mypy.overrides]]
+module = ["icu4py.*", "regex", "regex.*", "tokenizers", "onnxruntime"]
+ignore_missing_imports = true

memtrails-0.4.1/src/memtrails/__init__.py

@@ -0,0 +1,74 @@
+"""
+memtrails — retrieval and exploration for any text: e-mail, chat, notes, documents, news, commits, conversations.
+
+  add          records, conversations, or a stream from an importer — searchable at once, no model asked
+  search       dense + keyword, fused; reordered word by word on this machine; dates first when asked
+  room         the best sentences of what was found, for a small context
+  open         one item whole, with the records it was learned from
+  explore      the concept graph from a topic or items to what search does not find
+  consolidate  (with a language model) typed knowledge, dated observations, kept episodes
+  sleep        (with a language model) merge, supersede, link, profiles
+  gather       (with a language model) every instance a counting or listing question needs
+  tools        memory_search / memory_open / memory_explore for an agent
+
+A native port of the TypeScript library (npm ``memtrails``): the same algorithms, prompts and files.
+"""
+
+from .core.embedder import EmbedRoute
+from .core.gather import Gathered
+from .core.model import MODEL_REPO, MODEL_REVISION, default_model_dir, ensure_model
+from .core.paths import cache_dir
+from .core.qdrant_binary import QDRANT_VERSION, ensure_qdrant
+from .core.recall import Explored, RecalledItem, RecordRef
+from .core.reranker import RerankRoute
+from .core.room import sentence_room, sentence_spans
+from .core.words import WORD_LIMIT, PackedWords, WordStore, WordVectors, max_sim, pack_words
+from .llm import Ask, LlmRequest, LlmUsage, OpenAiAsk, openai_ask
+from .memory import EmbeddingOptions, Memory, QdrantOptions, SearchOptions, open_memory
+from .records import DEFAULT_COLLECTION, Conversation, MemoryRecord, record_uid, time_of
+from .tools.agent_tools import AgentTool, build_agent_tools, search_text
+
+__version__ = "0.4.1"
+
+__all__ = [
+    "DEFAULT_COLLECTION",
+    "MODEL_REPO",
+    "MODEL_REVISION",
+    "QDRANT_VERSION",
+    "WORD_LIMIT",
+    "AgentTool",
+    "Ask",
+    "Conversation",
+    "EmbedRoute",
+    "EmbeddingOptions",
+    "Explored",
+    "Gathered",
+    "LlmRequest",
+    "LlmUsage",
+    "Memory",
+    "MemoryRecord",
+    "OpenAiAsk",
+    "PackedWords",
+    "QdrantOptions",
+    "RecalledItem",
+    "RecordRef",
+    "RerankRoute",
+    "SearchOptions",
+    "WordStore",
+    "WordVectors",
+    "__version__",
+    "build_agent_tools",
+    "cache_dir",
+    "default_model_dir",
+    "ensure_model",
+    "ensure_qdrant",
+    "max_sim",
+    "open_memory",
+    "openai_ask",
+    "pack_words",
+    "record_uid",
+    "search_text",
+    "sentence_room",
+    "sentence_spans",
+    "time_of",
+]

memtrails-0.4.1/src/memtrails/_aio.py

@@ -0,0 +1,23 @@
+"""asyncio pieces the port needs on every Python it supports (3.10 has neither TaskGroup nor asyncio.timeout)."""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Awaitable
+from typing import TypeVar
+
+T = TypeVar("T")
+
+
+async def gather_or_cancel(*aws: Awaitable[T]) -> list[T]:
+    """``asyncio.gather``, but the first that raises cancels the others before it is raised, and they have ended when it
+    is: asks the model is paid for are not left running for an answer no one reads (a cancelled caller cancels them
+    too, as gather does)."""
+    tasks: list[asyncio.Future[T]] = [asyncio.ensure_future(a) for a in aws]
+    try:
+        return list(await asyncio.gather(*tasks))
+    except BaseException:
+        for t in tasks:
+            t.cancel()
+        await asyncio.gather(*tasks, return_exceptions=True)
+        raise