yaams 0.3.0__py3-none-any.whl

yaams/__init__.py

@@ -0,0 +1,6 @@
+"""YAAMS Phase A ingest package."""
+
+__all__ = ["__version__"]
+
+__version__ = "0.3.0"
+

yaams/_default_config.yaml

@@ -0,0 +1,129 @@
+# YAAMS configuration. Copy to config.yaml and fill in your own values.
+# See README.md and .plans/ for what each block does.
+
+db_path: ~/yaams/data.db
+
+ingest:
+  since: '2025-01-01T00:00:00Z'
+
+  imessage:
+    enabled: true
+    chat_db_path: ~/Library/Messages/chat.db
+
+  email:
+    enabled: true
+    sources:
+      # Apple Mail .emlx tree (point at the newest ~/Library/Mail/Vxx)
+      - type: emlx
+        path: ~/Library/Mail/V10
+      # Or one or more .mbox exports:
+      # - type: mbox
+      #   path: ~/Downloads/all-mail.mbox
+    skip_newsletters: true
+    # Emails From these addresses are always kept regardless of List-* headers
+    # or noreply patterns. Add every address you send from across roles.
+    user_addresses:
+      - you@example.com
+
+  # Personal Obsidian vault. One vault per source; honors frontmatter
+  # `date:` / `created:` or YYYY-MM-DD filename prefixes for timestamps,
+  # strips embeds/wikilinks/frontmatter before indexing. Set `vault_path`
+  # via the `yaams sources` TUI or by editing it here.
+  notes:
+    enabled: false
+    vault_path: ~/Documents/Obsidian
+    # Optional. Directory names to skip anywhere in the vault tree.
+    # Defaults to .obsidian, .git, .smartchats, .smart-env, .claude.
+    # skip_dirs: [.obsidian, .git, archive]
+
+  # Generic recursive folder ingestion. Reads .txt and .md natively;
+  # .pdf needs `pip install pypdf`, .docx needs `pip install python-docx`.
+  # Files of unsupported types (or types with the dep missing) are skipped.
+  # Paths can be added/removed from the `yaams sources` TUI.
+  folders:
+    enabled: false
+    paths: []
+      # - ~/Documents/notes
+      # - ~/work/specs
+    # Optional. Defaults to .txt, .md, .markdown, .pdf, .docx
+    # extensions: [.txt, .md, .pdf]
+    # Optional. Directory names to skip anywhere in the tree.
+    # skip_dirs: [.git, .obsidian, node_modules]
+
+  # ---- Microsoft 365 / Graph sources ----
+  # All M365 sources authenticate via owa-piggy. Configure profiles once with
+  # `owa-piggy setup --profile <name>` and reference them by alias below.
+  # `calendar` and `mail` are profile-keyed and produce one yaams source per
+  # profile (e.g. mail_work, calendar_work). `teams` likewise.
+
+  teams:
+    enabled: false
+    profiles:
+      - work
+    skip_bots: true
+    page_size: 50
+
+  # owa-mail-backed inbox + sent ingestion. One source per profile.
+  # `folders` defaults to [Inbox, SentItems] — add Archive/etc. if wanted.
+  mail:
+    enabled: false
+    profiles:
+      - work
+    folders:
+      - Inbox
+      - SentItems
+    skip_newsletters: true
+    # Date range is sliced into chunks to stay under owa-mail's 200/req cap.
+    chunk_days: 30
+    # Addresses you send from. Messages From these are never filtered as
+    # newsletters/automated even if subject patterns would otherwise match.
+    user_addresses: []
+      # - you@example.com
+
+  # calendar:
+  #   enabled: false
+  #   profiles:
+  #     - work
+  #   skip_free: true
+
+embed:
+  model: BAAI/bge-m3
+  batch_size: 32
+  # Apple Silicon: mps. Otherwise: cpu. CUDA also works if torch sees it.
+  device: mps
+  dimension: 1024
+  # Use only the locally cached HF snapshot; no network calls per run.
+  # Set to false the first time you change `model` so the new weights download.
+  offline: true
+  # Where HF stores model weights (sets HF_HOME). Defaults to
+  # ~/.local/share/huggingface so multi-GB weights survive `~/.cache` wipes.
+  # Respects an externally set $HF_HOME if you'd rather configure it that way.
+  # models_dir: ~/.local/share/huggingface
+
+# LLM backend for synthesis (and future query parsing).
+# Used by `yaams query --answer ...`. Off by default (dummy backend); pick
+# one of:
+#   backend: ollama       # local Ollama server
+#     model: llama3.1
+#     host: http://localhost:11434
+#   backend: subprocess   # any CLI that takes prompt on stdin and prints answer
+#     command: ["codex", "exec", "--prompt-stdin"]
+#     # or: ["claude", "-p"]
+#   backend: dummy        # no synthesis (also default if section omitted)
+synth:
+  backend: dummy
+
+entities:
+  spacy_model: xx_ent_wiki_sm
+  dictionary:
+    # Known people, places, projects, and aliases for entity tagging.
+    # Surface form on the left, casual variants under aliases.
+    - canonical: Example Person
+      type: person
+      aliases:
+        - Ex
+        - E. Person
+    - canonical: Example Org
+      type: org
+      aliases:
+        - EX

yaams/cli/__init__.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from yaams.cli import consolidate as _consolidate_mod  # noqa: F401
+from yaams.cli import assoc, enrich, entities, main, promote, query, review, signals, sources  # noqa: F401
+from yaams.cli import ingest as _ingest_mod  # noqa: F401
+from yaams.cli._root import cli
+from yaams.cli._shared import _format_duration
+from yaams.cli.consolidate import consolidate
+from yaams.cli.ingest import _record_ingest_run, ingest
+from yaams.cli.main import init_db, reset_db
+from yaams.cli.query import query_cmd
+
+__all__ = [
+  "cli",
+  "init_db",
+  "reset_db",
+  "ingest",
+  "query_cmd",
+  "consolidate",
+  "_format_duration",
+  "_record_ingest_run",
+]

yaams/cli/_envelope.py

@@ -0,0 +1,167 @@
+"""JSON failure envelope guard for YAAMS data-class commands.
+
+Background (Plan 06)
+--------------------
+Action-class commands (init-db, setup, ingest, reset-db) already wrap
+their ``load_config`` call in a try/except and emit an action envelope
+on failure. Data-class commands (``query``, ``stats``) historically
+called ``load_config`` outside any try block, so a missing or malformed
+config produced a raw Python traceback on stderr - exit 1 with no JSON
+on stdout. Hugr's passthrough wrapper sees that as "tool crashed" and
+hides the underlying config error from the user.
+
+This module gives data commands a single, uniform way to satisfy the
+hugr CLI contract for ``--json``: stdout is exactly one line of valid
+JSON, ok=false on failure, exit code mapped from CONVENTIONS.md.
+
+Usage
+-----
+    from yaams.cli._envelope import JsonFailureGuard
+
+    @cli.command(...)
+    def query(..., as_json: bool):
+      with JsonFailureGuard("query", as_json=as_json):
+        cfg = load_config(config_path)
+        ...  # rest of the command body
+
+The guard is a no-op when ``as_json`` is False - human-mode callers
+keep seeing the traceback so debugging stays easy.
+"""
+from __future__ import annotations
+
+import sqlite3
+import sys
+import traceback
+from contextlib import contextmanager
+from typing import Iterator, TextIO
+
+from yaams.conventions import (
+  EXIT_NOT_FOUND,
+  EXIT_USER_ERROR,
+  data_error,
+  emit_data_error,
+)
+
+
+def _classify(exc: BaseException) -> tuple[str, str | None, int]:
+  """Map an exception to (error_code, hint, exit_code).
+
+  Known classes get a stable code so hugr and other callers can branch
+  on it; everything else falls into ``unhandled`` with EXIT_USER_ERROR.
+
+  Exit codes follow CONVENTIONS.md:
+    - EXIT_NOT_FOUND (4) for missing config/db files
+    - EXIT_USER_ERROR (1) for malformed config and other user-recoverable
+      errors
+  """
+  # YAML parse errors land before FileNotFoundError checks because some
+  # yaml.YAMLError subclasses also subclass OSError on certain platforms;
+  # we want the parser error to take priority.
+  try:
+    import yaml  # local import: yaml is a runtime dep but importing
+                  # at module top wires it into every import chain.
+  except ImportError:  # pragma: no cover - PyYAML is required at runtime
+    yaml = None  # type: ignore[assignment]
+
+  if yaml is not None and isinstance(exc, yaml.YAMLError):
+    return (
+      "config_invalid",
+      "Fix the YAML syntax in your config file; `python -m yaml < path` "
+      "shows the parse error.",
+      EXIT_USER_ERROR,
+    )
+  if isinstance(exc, FileNotFoundError):
+    # Distinguish config-not-found from db-file-not-found by inspecting
+    # the missing-file path. ``FileNotFoundError`` carries the filename
+    # in ``.filename`` when raised via pathlib/open; fall back to the
+    # message for hand-raised cases.
+    target = getattr(exc, "filename", None) or str(exc)
+    target_lower = target.lower()
+    if (
+      target_lower.endswith(".yaml")
+      or target_lower.endswith(".yml")
+      or target_lower.endswith(".yaml'")
+      or target_lower.endswith(".yml'")
+      or "/yaams/config." in target_lower
+      or "/hugr/yaams/config." in target_lower
+      or "config" in target_lower
+    ):
+      return (
+        "config_not_found",
+        "Run `hugr init` to generate a config, or pass --config <path>.",
+        EXIT_NOT_FOUND,
+      )
+    return (
+      "db_open_failed",
+      "Run `yaams init-db` to create the database.",
+      EXIT_NOT_FOUND,
+    )
+  if isinstance(exc, sqlite3.OperationalError):
+    return (
+      "db_open_failed",
+      "Run `yaams init-db` or check the db_path in your config.",
+      EXIT_USER_ERROR,
+    )
+  if isinstance(exc, ValueError):
+    # ``Config file must contain a mapping`` and ``Config is missing
+    # db_path`` both surface as ValueError today.
+    return (
+      "config_invalid",
+      "Check your config.yaml against the example in the repo.",
+      EXIT_USER_ERROR,
+    )
+  return ("unhandled", None, EXIT_USER_ERROR)
+
+
+@contextmanager
+def JsonFailureGuard(
+  command: str,
+  *,
+  as_json: bool,
+  stdout: TextIO | None = None,
+  stderr: TextIO | None = None,
+) -> Iterator[None]:
+  """Wrap a data-command body so failures become JSON envelopes.
+
+  Parameters
+  ----------
+  command:
+    The command name as it appears in the envelope (e.g. ``"query"``,
+    ``"stats"``). Used verbatim, matching the existing action-envelope
+    convention.
+  as_json:
+    When False the guard is a no-op (the wrapped body runs and any
+    exception propagates normally). When True, exceptions inside the
+    block are caught, mapped to a ``data_error`` envelope, written as
+    one line of JSON to ``stdout``, and the process exits with the
+    code from ``_classify``.
+  stdout, stderr:
+    Optional injection points for tests. Default to ``sys.stdout`` and
+    ``sys.stderr``.
+
+  The body's own ``sys.exit`` / ``SystemExit`` is allowed to escape
+  unchanged - the guard only catches non-SystemExit exceptions.
+  """
+  if not as_json:
+    yield
+    return
+
+  out = stdout if stdout is not None else sys.stdout
+  err = stderr if stderr is not None else sys.stderr
+
+  try:
+    yield
+  except SystemExit:
+    raise
+  except BaseException as exc:  # noqa: BLE001 - guard is the catch-all
+    code, hint, exit_code = _classify(exc)
+    envelope = data_error(
+      command=command,
+      code=code,
+      message=str(exc),
+      hint=hint,
+    )
+    emit_data_error(envelope, stream=out)
+    # Traceback to stderr only, so stdout stays a single JSON line.
+    traceback.print_exception(type(exc), exc, exc.__traceback__, file=err)
+    sys.exit(exit_code)

yaams/cli/_root.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+import click
+
+from yaams import __version__
+
+
+@click.group(invoke_without_command=True)
+@click.version_option(__version__, prog_name="yaams")
+@click.option(
+  "--doctor",
+  is_flag=True,
+  default=False,
+  help="Run health check and exit (data class; pair with --json for machine output).",
+)
+@click.option(
+  "--json",
+  "as_json_top",
+  is_flag=True,
+  default=False,
+  help="Machine mode for top-level --doctor (subcommand form: `yaams doctor --json`).",
+)
+@click.option(
+  "--config",
+  "config_path_top",
+  default=None,
+  help="Path to config.yaml. Honored by top-level --doctor; subcommands take their own --config.",
+)
+@click.pass_context
+def cli(ctx: click.Context, doctor: bool, as_json_top: bool, config_path_top: str | None) -> None:
+  if doctor:
+    from yaams.cli.doctor import emit_doctor
+    ctx.exit(emit_doctor(config_path_top, as_json_top))
+  if ctx.invoked_subcommand is None:
+    click.echo(ctx.get_help())
+    ctx.exit(0)
+
+
+@cli.command("doctor")
+@click.option(
+  "--config",
+  "config_path",
+  default=None,
+  help="Path to config.yaml.",
+)
+@click.option(
+  "--json",
+  "as_json",
+  is_flag=True,
+  default=False,
+  help="Emit the doctor payload as JSON (machine mode).",
+)
+def doctor_cmd(config_path: str | None, as_json: bool) -> None:
+  """Run health check (subcommand alias for --doctor)."""
+  from yaams.cli.doctor import emit_doctor
+  exit_code = emit_doctor(config_path, as_json)
+  raise click.exceptions.Exit(exit_code)

yaams/cli/_shared.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Iterable
+
+import click
+
+from yaams.enrich import Embedder, EntityTagger
+from yaams.ingest import Item
+from yaams.schema import DEFAULT_EMBEDDING_DIM
+
+# Where HF model weights live by default. We keep them out of `~/.cache`
+# because they're durable, multi-GB artifacts, not regenerable cache.
+DEFAULT_MODELS_DIR = "~/.local/share/huggingface"
+
+_CONFIG_HELP = (
+  "Path to config.yaml. Auto-resolves from $YAAMS_CONFIG, "
+  "~/.config/yaams/config.yaml, or repo root if omitted."
+)
+
+
+def config_option(f):
+  return click.option("--config", "config_path", default=None, help=_CONFIG_HELP)(f)
+
+
+def _embed_config(cfg: dict) -> dict:
+  raw = dict(cfg.get("embed", {}))
+  model = raw.pop("model")
+  # Config wins; otherwise respect an externally set $HF_HOME; otherwise fall
+  # back to DEFAULT_MODELS_DIR so models survive `~/.cache` wipes.
+  if "models_dir" not in raw and not os.environ.get("HF_HOME"):
+    raw["models_dir"] = DEFAULT_MODELS_DIR
+  return {"model": model, **raw}
+
+
+def _embedding_dim(cfg: dict) -> int:
+  return int(cfg.get("embed", {}).get("dimension", DEFAULT_EMBEDDING_DIM))
+
+
+def _entities_config(cfg: dict) -> dict:
+  return dict(cfg.get("entities", {}))
+
+
+def _entity_dictionary(cfg: dict) -> list[dict]:
+  return list(_entities_config(cfg).get("dictionary", []))
+
+
+def _progress(iterable: Iterable[Item], desc: str, unit: str = "it") -> Iterable[Item]:
+  try:
+    from tqdm import tqdm
+
+    return tqdm(iterable, desc=desc, unit=unit)
+  except ImportError:
+    return iterable
+
+
+def _date(value: str | None) -> str:
+  if not value:
+    return "n/a"
+  return value[:10]
+
+
+def _size_mb(path: Path) -> float:
+  return path.stat().st_size / (1024 * 1024)
+
+
+def _format_duration(ms: float) -> str:
+  if ms < 1000:
+    return f"{ms:.0f}ms"
+  seconds = ms / 1000
+  if seconds < 60:
+    return f"{seconds:.1f}s"
+  minutes, seconds = divmod(seconds, 60)
+  return f"{int(minutes)}m{seconds:04.1f}s"
+
+
+def _format_throughput(seen: int, ms: float) -> str:
+  if ms <= 0 or seen <= 0:
+    return ""
+  rate = seen / (ms / 1000)
+  return f", {rate:,.1f} items/s"
+
+
+@dataclass
+class ProcessingContext:
+  cfg: dict
+  _embedder: Embedder | None = field(default=None, init=False)
+  _tagger: EntityTagger | None = field(default=None, init=False)
+
+  @property
+  def embedder(self) -> Embedder:
+    if self._embedder is None:
+      self._embedder = Embedder(**_embed_config(self.cfg))
+    return self._embedder
+
+  @property
+  def tagger(self) -> EntityTagger:
+    if self._tagger is None:
+      ent_cfg = _entities_config(self.cfg)
+      self._tagger = EntityTagger(
+        ent_cfg.get("spacy_model"),
+        _entity_dictionary(self.cfg),
+        spacy_model_nb=ent_cfg.get("spacy_model_nb"),
+      )
+    return self._tagger