sylo-fieldbrain 0.1.0

package/scripts/services/migrate_lock.py

@@ -0,0 +1,24 @@
+"""Postgres advisory lock while applying Alembic migrations (shared DB, many Sylo installs)."""
+
+from __future__ import annotations
+
+from contextlib import contextmanager
+
+from sqlalchemy import text
+from sqlalchemy.engine import Engine
+
+# Stable lock id for FieldBrain schema migrations.
+MIGRATE_LOCK_KEY = 74201904
+
+
+@contextmanager
+def migration_advisory_lock(engine: Engine):
+    with engine.connect() as conn:
+        conn.execute(text("SELECT pg_advisory_lock(:k)"), {"k": MIGRATE_LOCK_KEY})
+        conn.commit()
+    try:
+        yield
+    finally:
+        with engine.connect() as conn:
+            conn.execute(text("SELECT pg_advisory_unlock(:k)"), {"k": MIGRATE_LOCK_KEY})
+            conn.commit()

package/scripts/services/pgvector_windows.py

@@ -0,0 +1,190 @@
+"""Windows pgvector file install + setup hints for FieldBrain."""
+
+from __future__ import annotations
+
+import re
+import shutil
+import tempfile
+import zipfile
+from pathlib import Path
+from typing import Any
+
+from sqlalchemy import text
+from sqlalchemy.engine import Engine
+
+PREBUILT_RELEASES_PAGE = "https://github.com/andreiramani/pgvector_pgsql_windows/releases"
+
+
+def postgres_install_roots() -> list[tuple[int, Path]]:
+    """Installed PostgreSQL directories under Program Files (Windows)."""
+    roots: list[tuple[int, Path]] = []
+    for base in (Path(r"C:\Program Files\PostgreSQL"), Path(r"C:\Program Files (x86)\PostgreSQL")):
+        if not base.is_dir():
+            continue
+        for child in sorted(base.iterdir()):
+            if child.is_dir() and child.name.isdigit():
+                roots.append((int(child.name), child))
+    return roots
+
+
+def vector_control_path(pgroot: Path) -> Path:
+    return pgroot / "share" / "extension" / "vector.control"
+
+
+def vector_files_installed(pgroot: Path) -> bool:
+    return vector_control_path(pgroot).is_file()
+
+
+def detect_postgres_major(engine: Engine) -> int | None:
+    """Read server major version from a live connection."""
+    try:
+        with engine.connect() as conn:
+            row = conn.execute(text("SHOW server_version")).fetchone()
+        if not row or not row[0]:
+            return None
+        match = re.match(r"^(\d+)", str(row[0]))
+        return int(match.group(1)) if match else None
+    except Exception:
+        return None
+
+
+def guess_pgroot_for_major(major: int) -> Path | None:
+    for ver, root in postgres_install_roots():
+        if ver == major:
+            return root
+    return Path(rf"C:\Program Files\PostgreSQL\{major}")
+
+
+def resolve_source_dir(source: str | Path) -> tuple[Path | None, tempfile.TemporaryDirectory[str] | None, str | None]:
+    """Return a directory containing pgvector artifacts; extract zip when needed."""
+    path = Path(source).expanduser().resolve()
+    if not path.exists():
+        return None, None, f"Path not found: {path}"
+
+    if path.is_file() and path.suffix.lower() == ".zip":
+        tmp = tempfile.TemporaryDirectory(prefix="sylo-pgvector-")
+        with zipfile.ZipFile(path) as zf:
+            zf.extractall(tmp.name)
+        return Path(tmp.name), tmp, None
+
+    if path.is_dir():
+        return path, None, None
+
+    return None, None, f"Select a pgvector folder or .zip file: {path}"
+
+
+def _collect_pgvector_files(source_dir: Path) -> tuple[list[Path], list[Path], str | None]:
+    dlls: list[Path] = []
+    ext_files: list[Path] = []
+    for file in source_dir.rglob("*"):
+        if not file.is_file():
+            continue
+        name = file.name.lower()
+        if name.endswith(".dll"):
+            dlls.append(file)
+        elif name == "vector.control" or (name.startswith("vector--") and name.endswith(".sql")):
+            ext_files.append(file)
+    if not ext_files:
+        return [], [], "No vector.control or vector--*.sql found in the selected folder."
+    return dlls, ext_files, None
+
+
+def install_pgvector_files(source_dir: Path, pgroot: Path) -> dict[str, Any]:
+    """Copy pgvector binaries/extension SQL into a PostgreSQL install root."""
+    dlls, ext_files, err = _collect_pgvector_files(source_dir)
+    if err:
+        return {"ok": False, "error": err}
+
+    lib_dir = pgroot / "lib"
+    ext_dir = pgroot / "share" / "extension"
+    copied: list[str] = []
+
+    try:
+        lib_dir.mkdir(parents=True, exist_ok=True)
+        ext_dir.mkdir(parents=True, exist_ok=True)
+        for dll in dlls:
+            dest = lib_dir / dll.name
+            shutil.copy2(dll, dest)
+            copied.append(str(dest))
+        for ext in ext_files:
+            dest = ext_dir / ext.name
+            shutil.copy2(ext, dest)
+            copied.append(str(dest))
+    except PermissionError as exc:
+        return {
+            "ok": False,
+            "error": f"Permission denied copying into {pgroot}. Approve the admin prompt or run Sylo as administrator.",
+            "needs_elevation": True,
+            "pgroot": str(pgroot),
+            "source_dir": str(source_dir),
+        }
+    except OSError as exc:
+        return {"ok": False, "error": str(exc), "pgroot": str(pgroot)}
+
+    if not vector_files_installed(pgroot):
+        return {
+            "ok": False,
+            "error": f"Copy finished but {vector_control_path(pgroot)} is still missing.",
+            "copied": copied,
+            "pgroot": str(pgroot),
+        }
+
+    return {
+        "ok": True,
+        "copied": copied,
+        "pgroot": str(pgroot),
+        "vector_control_path": str(vector_control_path(pgroot)),
+    }
+
+
+def windows_install_status(postgres_major: int | None) -> dict[str, Any]:
+    """Whether pgvector files appear on disk for the detected major version."""
+    major = postgres_major
+    pgroot = guess_pgroot_for_major(major) if major else None
+    files_ok = bool(pgroot and vector_files_installed(pgroot))
+    return {
+        "postgres_major": major,
+        "pgroot": str(pgroot) if pgroot else None,
+        "vector_files_installed": files_ok,
+        "vector_control_path": str(vector_control_path(pgroot)) if pgroot else None,
+        "prebuilt_releases_page": PREBUILT_RELEASES_PAGE,
+    }
+
+
+def guided_pgvector_download_step(postgres_major: int | None) -> str:
+    major = postgres_major if postgres_major else "your Postgres major version"
+    return (
+        f"Optional — semantic search: download pgvector for PostgreSQL {major} from "
+        f"{PREBUILT_RELEASES_PAGE}, extract the zip, then use Browse below."
+    )
+
+
+def layman_pgvector_steps(
+    postgres_major: int | None,
+    *,
+    files_installed: bool,
+    db_mode: str = "local",
+) -> list[str]:
+    """Plain-language steps for optional semantic search."""
+    if db_mode == "remote":
+        return [
+            "Optional semantic search needs pgvector installed on the shared Postgres server (not this PC).",
+            guided_pgvector_download_step(postgres_major).replace("Browse below", "have your server admin install the files"),
+            "Then click Enable semantic search here (superuser password, not saved).",
+            "Test connection — pgvector should show enabled.",
+        ]
+
+    if files_installed:
+        return [
+            "pgvector files are on this PC.",
+            "Click Enable semantic search (superuser password above — not saved).",
+            "Test connection — pgvector should show enabled.",
+        ]
+
+    return [
+        "Optional: keyword search already works without pgvector.",
+        guided_pgvector_download_step(postgres_major),
+        "FieldBrain Settings → Browse → pick the extracted folder (or the .zip).",
+        "Click Install pgvector & enable semantic search (superuser password, not saved).",
+        "Test connection again.",
+    ]

package/scripts/services/project_naming.py

@@ -0,0 +1,72 @@
+"""Project naming: job 12345, sub-project 12345-001."""
+
+from __future__ import annotations
+
+import re
+
+JOB_PATTERN = re.compile(r"^(\d{5})$")
+SUB_SUFFIX_PATTERN = re.compile(r"^(\d{3})$")
+SUBJOB_PATTERN = re.compile(r"^(\d{5})-(\d{3})$")
+
+
+def normalize_digits(raw: str) -> str:
+    return raw.strip()
+
+
+def compose_subproject_name(job_number: str, sub_number: str) -> str:
+    return f"{job_number}-{sub_number}"
+
+
+def parse_job_number(name: str) -> str | None:
+    clean = normalize_digits(name)
+    job = JOB_PATTERN.match(clean)
+    if job:
+        return job.group(1)
+    sub = SUBJOB_PATTERN.match(clean)
+    if sub:
+        return sub.group(1)
+    return None
+
+
+def parse_subproject_suffix(raw: str) -> str | None:
+    """Accept 001 or 12345-001; return three-digit suffix."""
+    clean = normalize_digits(raw)
+    if not clean:
+        return None
+    suffix = SUB_SUFFIX_PATTERN.match(clean)
+    if suffix:
+        return suffix.group(1)
+    full = SUBJOB_PATTERN.match(clean)
+    if full:
+        return full.group(2)
+    return None
+
+
+def parse_project_name(name: str) -> tuple[str | None, str | None]:
+    """Return (job_number, sub_suffix) from a stored project name."""
+    clean = normalize_digits(name)
+    job = JOB_PATTERN.match(clean)
+    if job:
+        return job.group(1), None
+    sub = SUBJOB_PATTERN.match(clean)
+    if sub:
+        return sub.group(1), sub.group(2)
+    return None, None
+
+
+def validate_job_number(raw: str) -> str:
+    clean = normalize_digits(raw)
+    if not JOB_PATTERN.match(clean):
+        raise ValueError("Project number must be five digits (e.g. 12345).")
+    return clean
+
+
+def validate_subproject_number(raw: str, *, job_number: str) -> str:
+    clean = normalize_digits(raw)
+    suffix = parse_subproject_suffix(clean)
+    if suffix is None:
+        raise ValueError("Sub-project number must be three digits (e.g. 001) or full 12345-001.")
+    full = SUBJOB_PATTERN.match(clean)
+    if full and full.group(1) != job_number:
+        raise ValueError(f"Sub-project {clean!r} does not match project number {job_number}.")
+    return suffix

package/scripts/services/project_service.py

@@ -0,0 +1,368 @@
+"""Project CRUD helpers for sylo-fieldbrain."""
+
+from __future__ import annotations
+
+import json
+from typing import Any, Literal
+
+from sqlalchemy import func, or_
+from sqlalchemy.orm import Session
+
+from models import (
+    BrainDocument,
+    Document,
+    MaintenanceLogEntry,
+    Project,
+    ProjectDocumentLink,
+)
+from services.project_naming import (
+    JOB_PATTERN,
+    SUBJOB_PATTERN,
+    compose_subproject_name,
+    normalize_digits,
+    parse_job_number,
+    parse_project_name,
+    validate_job_number,
+    validate_subproject_number,
+)
+
+
+def _project_tags_list(project: Project) -> list[str]:
+    raw = getattr(project, "project_tags_json", None)
+    if not raw:
+        return []
+    try:
+        data = json.loads(raw)
+        if isinstance(data, list):
+            return [str(x) for x in data if str(x).strip()]
+    except json.JSONDecodeError:
+        pass
+    return []
+
+
+def _project_level(project: Project) -> Literal["job", "sub_project", "other"]:
+    name = project.name or ""
+    if project.parent_project_id is not None or SUBJOB_PATTERN.match(name):
+        return "sub_project"
+    if JOB_PATTERN.match(name):
+        return "job"
+    return "other"
+
+
+def project_to_dict(project: Project, *, stats: dict[str, int] | None = None) -> dict[str, Any]:
+    level = _project_level(project)
+    out: dict[str, Any] = {
+        "id": project.id,
+        "name": project.name,
+        "job_number": project.job_number,
+        "parent_project_id": project.parent_project_id,
+        "level": level,
+        "controller_name": project.controller_name,
+        "processor_type": project.processor_type,
+        "software_revision": project.software_revision,
+        "l5x_file_path": project.l5x_file_path,
+        "l5x_file_hash": project.l5x_file_hash,
+        "plc_ip": project.plc_ip,
+        "plc_read_enabled": bool(project.plc_read_enabled),
+        "org_id": project.org_id or "default",
+        "project_tags": _project_tags_list(project),
+        "l5x_status": project.l5x_status or "analyzed",
+        "created_at": project.created_at.isoformat() if project.created_at else None,
+        "updated_at": project.updated_at.isoformat() if project.updated_at else None,
+    }
+    if stats is not None:
+        out["stats"] = stats
+    return out
+
+
+def find_project_by_name(session: Session, name: str, *, org_id: str = "default") -> Project | None:
+    clean = normalize_digits(name)
+    if not clean:
+        return None
+    return (
+        session.query(Project)
+        .filter(
+            Project.org_id == (org_id or "default"),
+            func.lower(Project.name) == clean.lower(),
+        )
+        .first()
+    )
+
+
+def find_job_project(session: Session, job_number: str, *, org_id: str = "default") -> Project | None:
+    return find_project_by_name(session, job_number, org_id=org_id)
+
+
+def _brain_counts(session: Session) -> dict[int, int]:
+    rows = (
+        session.query(BrainDocument.project_id, func.count(BrainDocument.id))
+        .filter(
+            BrainDocument.deleted_at.is_(None),
+            BrainDocument.scope == "project",
+            BrainDocument.project_id.isnot(None),
+        )
+        .group_by(BrainDocument.project_id)
+        .all()
+    )
+    return {int(pid): int(count) for pid, count in rows if pid is not None}
+
+
+def _log_counts(session: Session) -> dict[int, int]:
+    rows = (
+        session.query(MaintenanceLogEntry.project_id, func.count(MaintenanceLogEntry.id))
+        .filter(
+            MaintenanceLogEntry.deleted_at.is_(None),
+            MaintenanceLogEntry.project_id.isnot(None),
+        )
+        .group_by(MaintenanceLogEntry.project_id)
+        .all()
+    )
+    return {int(pid): int(count) for pid, count in rows if pid is not None}
+
+
+def _library_doc_counts(session: Session) -> dict[int, int]:
+    linked = (
+        session.query(
+            ProjectDocumentLink.project_id.label("pid"),
+            ProjectDocumentLink.document_id.label("doc_id"),
+        )
+        .join(Document, Document.id == ProjectDocumentLink.document_id)
+        .filter(Document.archived.is_(False))
+    )
+    owned = session.query(
+        Document.owner_project_id.label("pid"),
+        Document.id.label("doc_id"),
+    ).filter(
+        Document.archived.is_(False),
+        Document.scope == "project_local",
+        Document.owner_project_id.isnot(None),
+    )
+    combined = linked.union_all(owned).subquery()
+    rows = (
+        session.query(combined.c.pid, func.count(func.distinct(combined.c.doc_id)))
+        .group_by(combined.c.pid)
+        .all()
+    )
+    return {int(pid): int(count) for pid, count in rows if pid is not None}
+
+
+def _stats_for_projects(session: Session, project_ids: list[int]) -> dict[int, dict[str, int]]:
+    if not project_ids:
+        return {}
+    brains = _brain_counts(session)
+    logs = _log_counts(session)
+    docs = _library_doc_counts(session)
+    out: dict[int, dict[str, int]] = {}
+    for pid in project_ids:
+        out[pid] = {
+            "brain_entries": brains.get(pid, 0),
+            "library_docs": docs.get(pid, 0),
+            "maintenance_logs": logs.get(pid, 0),
+        }
+    return out
+
+
+def _sort_projects(projects: list[Project]) -> list[Project]:
+    def key(p: Project) -> tuple[str, int, str]:
+        job = p.job_number or "zzzzz"
+        level_rank = 0 if _project_level(p) == "job" else 1
+        return (job, level_rank, p.name or "")
+
+    return sorted(projects, key=key)
+
+
+def list_projects(session: Session, *, include_hidden: bool = False) -> list[dict[str, Any]]:
+    query = session.query(Project)
+    if not include_hidden:
+        query = query.filter(or_(Project.is_system_hidden.is_(False), Project.is_system_hidden.is_(None)))
+    return [project_to_dict(p) for p in _sort_projects(query.all())]
+
+
+def list_projects_with_stats(session: Session, *, include_hidden: bool = False) -> list[dict[str, Any]]:
+    query = session.query(Project)
+    if not include_hidden:
+        query = query.filter(or_(Project.is_system_hidden.is_(False), Project.is_system_hidden.is_(None)))
+    projects = _sort_projects(query.all())
+    ids = [p.id for p in projects]
+    stats_map = _stats_for_projects(session, ids)
+    empty = {"brain_entries": 0, "library_docs": 0, "maintenance_logs": 0}
+    return [project_to_dict(p, stats=stats_map.get(p.id, empty)) for p in projects]
+
+
+def get_project(session: Session, project_id: int) -> dict[str, Any] | None:
+    project = session.query(Project).filter(Project.id == project_id).first()
+    if not project:
+        return None
+    stats = _stats_for_projects(session, [project.id]).get(project.id)
+    return project_to_dict(project, stats=stats)
+
+
+def _insert_project(
+    session: Session,
+    *,
+    name: str,
+    job_number: str,
+    parent_project_id: int | None,
+    project_tags: list[str] | None,
+    org_id: str,
+) -> Project:
+    tags_json = None
+    if project_tags:
+        cleaned = [str(t).strip() for t in project_tags if str(t).strip()]
+        if cleaned:
+            tags_json = json.dumps(cleaned)
+
+    project = Project(
+        name=name[:255],
+        job_number=job_number,
+        parent_project_id=parent_project_id,
+        project_tags_json=tags_json,
+        org_id=org_id,
+        l5x_status="pending",
+    )
+    session.add(project)
+    session.commit()
+    session.refresh(project)
+    return project
+
+
+def ensure_job_project(
+    session: Session,
+    job_number: str,
+    *,
+    org_id: str = "default",
+    project_tags: list[str] | None = None,
+) -> tuple[Project, bool]:
+    org = (org_id or "default").strip() or "default"
+    existing = find_job_project(session, job_number, org_id=org)
+    if existing:
+        return existing, False
+    project = _insert_project(
+        session,
+        name=job_number,
+        job_number=job_number,
+        parent_project_id=None,
+        project_tags=project_tags,
+        org_id=org,
+    )
+    return project, True
+
+
+def create_project_structured(
+    session: Session,
+    *,
+    job_number: str,
+    sub_project_number: str | None = None,
+    project_tags: list[str] | None = None,
+    org_id: str = "default",
+) -> dict[str, Any]:
+    """Create job-level project (12345) and/or sub-project (12345-001)."""
+    job = validate_job_number(job_number)
+    org = (org_id or "default").strip() or "default"
+    sub_raw = (sub_project_number or "").strip()
+
+    if not sub_raw:
+        existing = find_job_project(session, job, org_id=org)
+        if existing:
+            raise ValueError(
+                f"Project {job!r} already exists (id={existing.id}). "
+                "Use that id in chat or add a sub-project number."
+            )
+        project = _insert_project(
+            session,
+            name=job,
+            job_number=job,
+            parent_project_id=None,
+            project_tags=project_tags,
+            org_id=org,
+        )
+    else:
+        suffix = validate_subproject_number(sub_raw, job_number=job)
+        full_name = compose_subproject_name(job, suffix)
+        existing = find_project_by_name(session, full_name, org_id=org)
+        if existing:
+            raise ValueError(
+                f"Sub-project {full_name!r} already exists (id={existing.id}). "
+                "Use that project or pick a different sub-project number."
+            )
+        parent, _ = ensure_job_project(session, job, org_id=org, project_tags=None)
+        project = _insert_project(
+            session,
+            name=full_name,
+            job_number=job,
+            parent_project_id=parent.id,
+            project_tags=project_tags,
+            org_id=org,
+        )
+
+    result = project_to_dict(
+        project,
+        stats={"brain_entries": 0, "library_docs": 0, "maintenance_logs": 0},
+    )
+    if sub_raw:
+        result["parent_job_id"] = project.parent_project_id
+    else:
+        result["created_job_project"] = True
+    return result
+
+
+def create_project(
+    session: Session,
+    *,
+    name: str,
+    domain: str | None = None,
+    controller_family: str | None = None,
+    project_tags: list[str] | None = None,
+    org_id: str = "default",
+) -> dict[str, Any]:
+    """Agent/chat create by full name: 12345 or 12345-001."""
+    clean_name = normalize_digits(name)
+    if not clean_name:
+        raise ValueError("Project name is required")
+
+    job, suffix = parse_project_name(clean_name)
+    if job and suffix:
+        return create_project_structured(
+            session,
+            job_number=job,
+            sub_project_number=suffix,
+            project_tags=project_tags,
+            org_id=org_id,
+        )
+    if job and not suffix:
+        return create_project_structured(
+            session,
+            job_number=job,
+            sub_project_number=None,
+            project_tags=project_tags,
+            org_id=org_id,
+        )
+
+    org = (org_id or "default").strip() or "default"
+    existing = find_project_by_name(session, clean_name, org_id=org)
+    if existing:
+        raise ValueError(
+            f"Project name {clean_name!r} already exists (id={existing.id}). "
+            "Use five-digit job (12345) or sub-project (12345-001)."
+        )
+
+    project = _insert_project(
+        session,
+        name=clean_name[:255],
+        job_number=parse_job_number(clean_name),
+        parent_project_id=None,
+        project_tags=project_tags,
+        org_id=org,
+    )
+    if domain or controller_family:
+        if domain:
+            project.domain = domain.strip() or None
+        if controller_family:
+            project.controller_family = controller_family.strip() or None
+        session.commit()
+        session.refresh(project)
+
+    return project_to_dict(
+        project,
+        stats={"brain_entries": 0, "library_docs": 0, "maintenance_logs": 0},
+    )