sylo-fieldbrain 0.1.0

package/scripts/alembic.ini

@@ -0,0 +1,38 @@
+[alembic]
+script_location = alembic
+prepend_sys_path = .
+
+sqlalchemy.url = postgresql://logicscout:logicscout@localhost:5432/logicscout
+
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

package/scripts/db_auto_migrate.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+"""Apply Alembic migrations when the shared DB is behind this package (startup + Settings button)."""
+
+from __future__ import annotations
+
+import argparse
+
+from sqlalchemy import text
+
+from _db_lib import SCHEMA_VERSION, get_engine, read_applied_schema_version, reset_engine
+from _json_out import emit, emit_error
+from db_migrate import apply_migrations
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Report whether migration would run without changing the database",
+    )
+    args = parser.parse_args()
+
+    reset_engine()
+    try:
+        engine = get_engine()
+        with engine.connect() as conn:
+            conn.execute(text("SELECT 1"))
+    except Exception as exc:
+        emit(
+            {
+                "ok": True,
+                "migrated": False,
+                "skipped": True,
+                "reason": "not_connected",
+                "error": str(exc),
+                "operator_chat": f"Auto-migrate skipped: could not connect to Postgres ({exc}).",
+            }
+        )
+        return
+
+    try:
+        applied = read_applied_schema_version()
+    except Exception as exc:
+        if args.dry_run:
+            emit_error(f"Could not read schema version: {exc}")
+        applied = None
+
+    if applied == SCHEMA_VERSION:
+        emit(
+            {
+                "ok": True,
+                "migrated": False,
+                "already_current": True,
+                "applied_schema_version": applied,
+                "expected_schema_version": SCHEMA_VERSION,
+                "operator_chat": f"Database already at schema version {applied}.",
+            }
+        )
+        return
+
+    if applied is not None and applied > SCHEMA_VERSION:
+        emit_error(
+            f"Database schema {applied} is newer than this Sylo build ({SCHEMA_VERSION}). "
+            "Update Sylo on this machine (git pull / reinstall).",
+            applied_schema_version=applied,
+            expected_schema_version=SCHEMA_VERSION,
+        )
+
+    if args.dry_run:
+        emit(
+            {
+                "ok": True,
+                "migrated": False,
+                "would_migrate": True,
+                "applied_schema_version": applied,
+                "expected_schema_version": SCHEMA_VERSION,
+                "operator_chat": (
+                    f"Database at version {applied}; package expects {SCHEMA_VERSION}. "
+                    "Migration runs on Sylo startup or via Apply database updates in Settings."
+                ),
+            }
+        )
+        return
+
+    try:
+        new_version = apply_migrations()
+    except Exception as exc:
+        emit_error(f"Migration failed: {exc}")
+
+    emit(
+        {
+            "ok": True,
+            "migrated": True,
+            "applied_schema_version": new_version,
+            "expected_schema_version": SCHEMA_VERSION,
+            "operator_chat": f"FieldBrain database updated to schema version {new_version}.",
+        }
+    )
+
+
+if __name__ == "__main__":
+    main()

package/scripts/db_bootstrap.py

@@ -0,0 +1,207 @@
+#!/usr/bin/env python3
+"""One-time Postgres bootstrap: create FieldBrain role + database, pgvector, migrate."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import urllib.parse
+
+from sqlalchemy import create_engine, text
+
+from _db_lib import SCHEMA_VERSION, read_applied_schema_version, reset_engine
+from _json_out import emit, emit_error
+from db_migrate import alembic_config
+
+
+def _quote_ident(name: str) -> str:
+    return '"' + name.replace('"', '""') + '"'
+
+
+def _quote_literal(value: str) -> str:
+    return "'" + value.replace("'", "''") + "'"
+
+
+def _build_url(host: str, port: int, database: str, username: str, password: str) -> str:
+    safe_password = urllib.parse.quote_plus(password) if password else ""
+    auth = f"{urllib.parse.quote_plus(username)}:{safe_password}@" if safe_password else f"{urllib.parse.quote_plus(username)}@"
+    return f"postgresql://{auth}{host}:{port}/{database}"
+
+
+def _role_exists(conn, role: str) -> bool:
+    row = conn.execute(
+        text("SELECT 1 FROM pg_roles WHERE rolname = :name"),
+        {"name": role},
+    ).fetchone()
+    return row is not None
+
+
+def _database_exists(conn, database: str) -> bool:
+    row = conn.execute(
+        text("SELECT 1 FROM pg_database WHERE datname = :name"),
+        {"name": database},
+    ).fetchone()
+    return row is not None
+
+
+def _try_pgvector(app_url: str) -> tuple[bool, str | None]:
+    engine = create_engine(app_url, isolation_level="AUTOCOMMIT", pool_pre_ping=True)
+    try:
+        with engine.connect() as conn:
+            conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector"))
+        with engine.connect() as conn:
+            row = conn.execute(
+                text("SELECT extversion FROM pg_extension WHERE extname = 'vector'")
+            ).fetchone()
+        if row:
+            return True, str(row[0])
+        return False, "CREATE EXTENSION succeeded but vector not visible"
+    except Exception as exc:
+        return False, str(exc)
+    finally:
+        engine.dispose()
+
+
+def _run_migrate(app_url: str) -> tuple[int | None, str | None]:
+    from alembic import command
+
+    os.environ["SYLO_FIELDBRAIN_DATABASE_URL"] = app_url
+    reset_engine()
+    try:
+        command.upgrade(alembic_config(), "head")
+    except Exception as exc:
+        return None, str(exc)
+    reset_engine()
+    try:
+        applied = read_applied_schema_version()
+    except Exception as exc:
+        return None, f"Migration ran but schema version unreadable: {exc}"
+    if applied != SCHEMA_VERSION:
+        return applied, f"Expected schema {SCHEMA_VERSION}, got {applied}"
+    return applied, None
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--admin-user", default="postgres")
+    parser.add_argument("--admin-password", default="")
+    parser.add_argument("--host", default="localhost")
+    parser.add_argument("--port", type=int, default=5432)
+    parser.add_argument("--app-database", default="fieldbrain")
+    parser.add_argument("--app-username", default="fieldbrain")
+    parser.add_argument("--app-password", default="fieldbrain")
+    parser.add_argument("--skip-migrate", action="store_true")
+    args = parser.parse_args()
+
+    admin_password = (args.admin_password or os.environ.get("SYLO_FIELDBRAIN_BOOTSTRAP_ADMIN_PASSWORD") or "").strip()
+    if not admin_password:
+        emit_error("Postgres superuser password is required for bootstrap (not stored after this run).")
+
+    host = args.host.strip() or "localhost"
+    port = int(args.port)
+    app_db = args.app_database.strip() or "fieldbrain"
+    app_user = args.app_username.strip() or "fieldbrain"
+    app_password = args.app_password if args.app_password is not None else "fieldbrain"
+
+    admin_user = args.admin_user.strip() or "postgres"
+    admin_url = _build_url(host, port, "postgres", admin_user, admin_password)
+    steps: list[str] = []
+
+    try:
+        admin_engine = create_engine(admin_url, isolation_level="AUTOCOMMIT", pool_pre_ping=True)
+        with admin_engine.connect() as conn:
+            conn.execute(text("SELECT 1"))
+        steps.append(f"Connected as superuser {admin_user}@{host}:{port}")
+    except Exception as exc:
+        emit_error(
+            f"Could not connect as Postgres superuser: {exc}",
+            hint="Use the password you set at PostgreSQL install (or reset via pg_hba.conf trust).",
+        )
+
+    try:
+        with admin_engine.connect() as conn:
+            if _role_exists(conn, app_user):
+                conn.execute(
+                    text(
+                        f"ALTER USER {_quote_ident(app_user)} "
+                        f"WITH PASSWORD {_quote_literal(app_password)}"
+                    )
+                )
+                steps.append(f"Role {app_user} already existed — password updated")
+            else:
+                conn.execute(
+                    text(
+                        f"CREATE USER {_quote_ident(app_user)} "
+                        f"WITH PASSWORD {_quote_literal(app_password)}"
+                    )
+                )
+                steps.append(f"Created role {app_user}")
+
+            if _database_exists(conn, app_db):
+                steps.append(f"Database {app_db} already existed — skipped CREATE DATABASE")
+            else:
+                conn.execute(
+                    text(
+                        f"CREATE DATABASE {_quote_ident(app_db)} "
+                        f"OWNER {_quote_ident(app_user)}"
+                    )
+                )
+                steps.append(f"Created database {app_db}")
+    except Exception as exc:
+        admin_engine.dispose()
+        emit_error(f"Bootstrap DDL failed: {exc}", steps=steps)
+    admin_engine.dispose()
+
+    app_url = _build_url(host, port, app_db, app_user, app_password)
+    pgvector_ok, pgvector_detail = _try_pgvector(app_url)
+    if pgvector_ok:
+        steps.append(f"pgvector enabled ({pgvector_detail})")
+    else:
+        steps.append(
+            "pgvector not installed — semantic search unavailable until you install pgvector "
+            f"and run CREATE EXTENSION vector ({pgvector_detail})"
+        )
+
+    applied_version: int | None = None
+    if not args.skip_migrate:
+        applied_version, migrate_err = _run_migrate(app_url)
+        if migrate_err:
+            emit_error(
+                migrate_err,
+                steps=steps,
+                pgvector_ok=pgvector_ok,
+                app_database=app_db,
+                app_username=app_user,
+            )
+        steps.append(f"Migrated to schema version {applied_version}")
+
+    lines = [
+        f"FieldBrain database ready: {app_user}@{host}:{port}/{app_db}",
+        *steps,
+        "Superuser password was not saved. Runtime uses the FieldBrain user only.",
+    ]
+    if not pgvector_ok:
+        lines.append(
+            "Optional semantic search: download pgvector, browse to the folder in FieldBrain Settings, "
+            "then Install pgvector & enable semantic search."
+        )
+
+    emit(
+        {
+            "ok": True,
+            "operator_chat": "\n".join(lines),
+            "steps": steps,
+            "host": host,
+            "port": port,
+            "app_database": app_db,
+            "app_username": app_user,
+            "pgvector_ok": pgvector_ok,
+            "pgvector_detail": pgvector_detail,
+            "applied_schema_version": applied_version,
+            "expected_schema_version": SCHEMA_VERSION,
+        }
+    )
+
+
+if __name__ == "__main__":
+    main()

package/scripts/db_check.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+"""Postgres + pgvector + schema version check for fieldbrain_db_check tool."""
+
+from __future__ import annotations
+
+import argparse
+
+from _db_lib import SCHEMA_VERSION, connection_summary, run_health_check
+from _json_out import emit, emit_error
+from services.pgvector_windows import layman_pgvector_steps
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--guided",
+        action="store_true",
+        help="Include guided setup steps when checks fail",
+    )
+    args = parser.parse_args()
+
+    summary = run_health_check()
+    connected = summary.get("postgres_connected") is True
+    schema_ok = summary.get("schema_ok") is True
+    pgvector_ok = summary.get("pgvector_available") is True
+
+    if not connected:
+        payload = {
+            "ok": False,
+            "error": summary.get("postgres_error", "Could not connect to PostgreSQL"),
+            **summary,
+        }
+        if args.guided:
+            payload["guided_setup"] = _guided_postgres_steps()
+        emit(payload)
+
+    lines = [
+        f"PostgreSQL: connected ({summary.get('database_url_host')}:{summary.get('database_url_port')}/{summary.get('database_url_name')})",
+        f"Config: {summary.get('config_source')} ({summary.get('config_file')})",
+        f"Schema: {summary.get('schema_message')}",
+        f"pgvector: {'enabled' if pgvector_ok else 'missing — semantic search unavailable until installed'}",
+        f"Ollama endpoint: {summary.get('ollama_url')}",
+    ]
+
+    if not schema_ok:
+        applied = summary.get("applied_schema_version")
+        expected = summary.get("expected_schema_version", SCHEMA_VERSION)
+        needs_migration = (
+            summary.get("postgres_connected") is True
+            and isinstance(applied, int)
+            and applied < expected
+        ) or (
+            summary.get("postgres_connected") is True and applied is None
+        )
+        payload = {
+            "ok": False,
+            "error": summary.get("schema_message"),
+            "operator_chat": "\n".join(lines),
+            "needs_migration": needs_migration,
+            **summary,
+        }
+        if args.guided:
+            payload["guided_setup"] = _guided_migrate_steps()
+        emit(payload)
+
+    payload: dict = {
+        "ok": True,
+        "operator_chat": "\n".join(lines),
+        **summary,
+        "expected_schema_version": SCHEMA_VERSION,
+    }
+    if not pgvector_ok:
+        major = summary.get("postgres_major")
+        payload["pgvector_setup"] = layman_pgvector_steps(
+            major if isinstance(major, int) else None,
+            files_installed=bool(summary.get("vector_files_installed")),
+            db_mode="remote"
+            if str(summary.get("database_url_host") or "") not in ("localhost", "127.0.0.1")
+            else "local",
+        )
+    emit(payload)
+
+
+def _guided_postgres_steps() -> list[str]:
+    return [
+        "Install PostgreSQL 13+ as a Windows service — download from https://www.postgresql.org/download/windows/",
+        "Optional — semantic search: download pgvector for your Postgres major version from "
+        "https://github.com/andreiramani/pgvector_pgsql_windows/releases and extract the zip.",
+        "FieldBrain Settings → Create database & migrate (button; superuser password not saved).",
+        "Optional: Browse to the pgvector folder or zip → Install pgvector & enable semantic search.",
+        "Test connection. Shop laptops: set dbMode remote + server IP.",
+    ]
+
+
+def _guided_migrate_steps() -> list[str]:
+    return [
+        "Sylo auto-applies database updates on startup when FieldBrain is enabled (one machine is enough for a shared DB).",
+        "Or FieldBrain Settings → Apply database updates (next to Test connection).",
+        "Or chat: fieldbrain_db_migrate.",
+        "After migrate succeeds, restart other Sylo installs (they pick up the new schema automatically).",
+    ]
+
+
+if __name__ == "__main__":
+    main()

package/scripts/db_migrate.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+"""Apply Alembic migrations (fieldbrain_db_migrate tool)."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+from alembic import command
+from alembic.config import Config
+
+from _db_lib import SCHEMA_VERSION, build_database_url, read_applied_schema_version, reset_engine
+from _json_out import emit, emit_error
+from services.migrate_lock import migration_advisory_lock
+
+
+def alembic_config() -> Config:
+    scripts_dir = Path(__file__).resolve().parent
+    cfg = Config(str(scripts_dir / "alembic.ini"))
+    cfg.set_main_option("script_location", str(scripts_dir / "alembic"))
+    cfg.set_main_option("sqlalchemy.url", build_database_url())
+    return cfg
+
+
+def apply_migrations() -> int:
+    """Upgrade to head under advisory lock; return applied schema version."""
+    reset_engine()
+    from _db_lib import get_engine
+
+    engine = get_engine()
+    with migration_advisory_lock(engine):
+        command.upgrade(alembic_config(), "head")
+    reset_engine()
+    applied = read_applied_schema_version()
+    if applied != SCHEMA_VERSION:
+        raise RuntimeError(
+            f"Migration finished but schema version is {applied}, expected {SCHEMA_VERSION}."
+        )
+    return int(applied)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--check-only",
+        action="store_true",
+        help="Report current schema version without migrating",
+    )
+    args = parser.parse_args()
+
+    reset_engine()
+    url = build_database_url()
+
+    if args.check_only:
+        try:
+            applied = read_applied_schema_version()
+        except Exception as exc:
+            emit_error(f"Could not read schema version: {exc}", database_url_host=url)
+        emit(
+            {
+                "ok": True,
+                "applied_schema_version": applied,
+                "expected_schema_version": SCHEMA_VERSION,
+                "operator_chat": f"Applied schema version: {applied}; package expects {SCHEMA_VERSION}.",
+            }
+        )
+
+    try:
+        applied = apply_migrations()
+    except Exception as exc:
+        emit_error(f"Migration failed: {exc}")
+
+    emit(
+        {
+            "ok": True,
+            "migrated": True,
+            "applied_schema_version": applied,
+            "expected_schema_version": SCHEMA_VERSION,
+            "operator_chat": f"FieldBrain database migrated to schema version {applied}.",
+        }
+    )
+
+
+if __name__ == "__main__":
+    main()

package/scripts/fieldbrain_brain_delete.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+"""Soft-delete a brain document (logicscout_brain_delete tool)."""
+
+from __future__ import annotations
+
+import argparse
+
+from _db_lib import get_engine, verify_schema_version
+from _json_out import emit, emit_error
+from services.brain_service import soft_delete_brain
+from sqlalchemy.orm import Session
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--path", required=True)
+    parser.add_argument("--scope", choices=("global", "project"), default="project")
+    parser.add_argument("--project-id", type=int, default=None)
+    args = parser.parse_args()
+
+    ok, msg = verify_schema_version()
+    if not ok:
+        emit_error(msg)
+
+    if args.scope == "project" and args.project_id is None:
+        emit_error("--project-id is required when scope is project")
+
+    try:
+        with Session(get_engine()) as session:
+            result = soft_delete_brain(
+                session,
+                scope=args.scope,
+                relative_path=args.path,
+                project_id=args.project_id,
+            )
+    except ValueError as exc:
+        emit_error(str(exc))
+
+    emit(
+        {
+            "ok": True,
+            **result,
+            "operator_chat": f"Soft-deleted brain {args.scope}:{args.path} (revision saved).",
+        }
+    )
+
+
+if __name__ == "__main__":
+    main()

package/scripts/fieldbrain_brain_read.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+"""Read brain markdown (logicscout_brain_read tool)."""
+
+from __future__ import annotations
+
+import argparse
+
+from _db_lib import get_engine, verify_schema_version
+from _json_out import emit, emit_error
+from services.brain_service import read_brain_markdown
+from sqlalchemy.orm import Session
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--path", required=True, help="Relative brain path, e.g. gotchas/conveyor_jam.md")
+    parser.add_argument(
+        "--scope",
+        choices=("global", "project"),
+        default="project",
+        help="global org brain or project brain",
+    )
+    parser.add_argument("--project-id", type=int, default=None, help="Required when scope=project")
+    args = parser.parse_args()
+
+    ok, msg = verify_schema_version()
+    if not ok:
+        emit_error(msg)
+
+    if args.scope == "project" and args.project_id is None:
+        emit_error("--project-id is required when scope is project")
+
+    try:
+        with Session(get_engine()) as session:
+            result = read_brain_markdown(
+                session,
+                scope=args.scope,
+                relative_path=args.path,
+                project_id=args.project_id,
+            )
+    except ValueError as exc:
+        emit_error(str(exc))
+
+    preview = result["content"]
+    if len(preview) > 1200:
+        preview = preview[:1200] + "\n… (truncated in chat; full content in JSON)"
+
+    emit(
+        {
+            "ok": True,
+            **result,
+            "operator_chat": (
+                f"Brain {result['scope']}:{result['relative_path']} "
+                f"({result['size_bytes']} bytes)\n\n{preview}"
+            ),
+        }
+    )
+
+
+if __name__ == "__main__":
+    main()