memtrails 0.4.1__py3-none-any.whl

memtrails/__init__.py

@@ -0,0 +1,74 @@
+"""
+memtrails — retrieval and exploration for any text: e-mail, chat, notes, documents, news, commits, conversations.
+
+  add          records, conversations, or a stream from an importer — searchable at once, no model asked
+  search       dense + keyword, fused; reordered word by word on this machine; dates first when asked
+  room         the best sentences of what was found, for a small context
+  open         one item whole, with the records it was learned from
+  explore      the concept graph from a topic or items to what search does not find
+  consolidate  (with a language model) typed knowledge, dated observations, kept episodes
+  sleep        (with a language model) merge, supersede, link, profiles
+  gather       (with a language model) every instance a counting or listing question needs
+  tools        memory_search / memory_open / memory_explore for an agent
+
+A native port of the TypeScript library (npm ``memtrails``): the same algorithms, prompts and files.
+"""
+
+from .core.embedder import EmbedRoute
+from .core.gather import Gathered
+from .core.model import MODEL_REPO, MODEL_REVISION, default_model_dir, ensure_model
+from .core.paths import cache_dir
+from .core.qdrant_binary import QDRANT_VERSION, ensure_qdrant
+from .core.recall import Explored, RecalledItem, RecordRef
+from .core.reranker import RerankRoute
+from .core.room import sentence_room, sentence_spans
+from .core.words import WORD_LIMIT, PackedWords, WordStore, WordVectors, max_sim, pack_words
+from .llm import Ask, LlmRequest, LlmUsage, OpenAiAsk, openai_ask
+from .memory import EmbeddingOptions, Memory, QdrantOptions, SearchOptions, open_memory
+from .records import DEFAULT_COLLECTION, Conversation, MemoryRecord, record_uid, time_of
+from .tools.agent_tools import AgentTool, build_agent_tools, search_text
+
+__version__ = "0.4.1"
+
+__all__ = [
+    "DEFAULT_COLLECTION",
+    "MODEL_REPO",
+    "MODEL_REVISION",
+    "QDRANT_VERSION",
+    "WORD_LIMIT",
+    "AgentTool",
+    "Ask",
+    "Conversation",
+    "EmbedRoute",
+    "EmbeddingOptions",
+    "Explored",
+    "Gathered",
+    "LlmRequest",
+    "LlmUsage",
+    "Memory",
+    "MemoryRecord",
+    "OpenAiAsk",
+    "PackedWords",
+    "QdrantOptions",
+    "RecalledItem",
+    "RecordRef",
+    "RerankRoute",
+    "SearchOptions",
+    "WordStore",
+    "WordVectors",
+    "__version__",
+    "build_agent_tools",
+    "cache_dir",
+    "default_model_dir",
+    "ensure_model",
+    "ensure_qdrant",
+    "max_sim",
+    "open_memory",
+    "openai_ask",
+    "pack_words",
+    "record_uid",
+    "search_text",
+    "sentence_room",
+    "sentence_spans",
+    "time_of",
+]

memtrails/_aio.py

@@ -0,0 +1,23 @@
+"""asyncio pieces the port needs on every Python it supports (3.10 has neither TaskGroup nor asyncio.timeout)."""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Awaitable
+from typing import TypeVar
+
+T = TypeVar("T")
+
+
+async def gather_or_cancel(*aws: Awaitable[T]) -> list[T]:
+    """``asyncio.gather``, but the first that raises cancels the others before it is raised, and they have ended when it
+    is: asks the model is paid for are not left running for an answer no one reads (a cancelled caller cancels them
+    too, as gather does)."""
+    tasks: list[asyncio.Future[T]] = [asyncio.ensure_future(a) for a in aws]
+    try:
+        return list(await asyncio.gather(*tasks))
+    except BaseException:
+        for t in tasks:
+            t.cancel()
+        await asyncio.gather(*tasks, return_exceptions=True)
+        raise

memtrails/_fs.py

@@ -0,0 +1,44 @@
+"""Writes that land whole or raise: ``os.write`` may write fewer bytes than it was given (a full disk, a signal), and a
+short write left unnoticed tears a line, or leaves the offsets an index holds pointing past the file's end."""
+
+from __future__ import annotations
+
+import errno
+import os
+
+
+def write_all(fd: int, data: bytes) -> None:
+    """Every byte of ``data`` to ``fd``, or OSError."""
+    view = memoryview(data)
+    while view:
+        n = os.write(fd, view)
+        if n <= 0:
+            raise OSError(errno.EIO, "nothing was written")
+        view = view[n:]
+
+
+def append_all(path: str, data: bytes) -> None:
+    """``data`` appended to ``path`` (made private when it is new), whole — or OSError, and the file cut back to where
+    it ended before, so what is appended next starts on a line of its own."""
+    fd = os.open(path, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o600)
+    try:
+        start = os.fstat(fd).st_size
+        try:
+            write_all(fd, data)
+        except OSError:
+            try:
+                os.ftruncate(fd, start)
+            except OSError:
+                pass  # what was cut short stays: a load ends or drops a torn last line
+            raise
+    finally:
+        os.close(fd)
+
+
+def write_file(path: str, data: bytes) -> None:
+    """``path`` replaced by ``data`` (private), whole, or OSError."""
+    fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+    try:
+        write_all(fd, data)
+    finally:
+        os.close(fd)

memtrails/_http.py

@@ -0,0 +1,25 @@
+"""One HTTP client for everything a memory talks to (Qdrant, the embedding and rerank servers, the language model, the
+downloads): the caller's, or one made for the call and closed after it when there is none. Never the environment's
+proxies — memory's own Qdrant is on loopback, and a key meant for a server must reach that server."""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+
+import httpx
+
+
+def new_client() -> httpx.AsyncClient:
+    """The client a memory makes for itself when it is given none."""
+    return httpx.AsyncClient(trust_env=False, timeout=httpx.Timeout(30.0))
+
+
+@asynccontextmanager
+async def using(http: httpx.AsyncClient | None) -> AsyncIterator[httpx.AsyncClient]:
+    """``http`` when given; else a client of its own for this call."""
+    if http is not None:
+        yield http
+        return
+    async with new_client() as own:
+        yield own