sylo-fieldbrain 0.1.0

package/scripts/services/document_service.py

@@ -0,0 +1,316 @@
+"""Document metadata queries and attach/register helpers for sylo-logicscout."""
+
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+from typing import Any
+
+from sqlalchemy import or_, text
+from sqlalchemy.orm import Session
+
+from models import Document, Project, ProjectDocumentLink
+from services.brain_paths import storage_root
+from services.document_formats import (
+    ALLOWED_EXTENSIONS,
+    MAX_FILE_SIZE,
+    mime_for_ext,
+    normalize_category,
+)
+
+
+def _file_hash(path: Path) -> str:
+    hasher = hashlib.sha256()
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(65536), b""):
+            hasher.update(chunk)
+    return hasher.hexdigest()
+
+
+def document_to_dict(document: Document, *, link_role: str | None = None) -> dict[str, Any]:
+    return {
+        "id": document.id,
+        "scope": document.scope,
+        "owner_project_id": document.owner_project_id,
+        "title": document.title,
+        "original_filename": document.original_filename,
+        "stored_path": document.stored_path,
+        "file_hash": document.file_hash,
+        "mime_type": document.mime_type,
+        "file_size": document.file_size,
+        "category": document.category,
+        "manufacturer": document.manufacturer,
+        "model": document.model,
+        "version": document.version,
+        "archived": bool(document.archived),
+        "created_at": document.created_at.isoformat() if document.created_at else None,
+        "tags": document.tags_json,
+        "link_role": link_role,
+    }
+
+
+def get_document(session: Session, document_id: int) -> dict[str, Any] | None:
+    document = (
+        session.query(Document)
+        .filter(Document.id == document_id, Document.archived.is_(False))
+        .first()
+    )
+    if not document:
+        return None
+    return document_to_dict(document)
+
+
+def list_documents_for_project(session: Session, project_id: int) -> list[dict[str, Any]]:
+    rows = (
+        session.query(Document, ProjectDocumentLink)
+        .join(ProjectDocumentLink, ProjectDocumentLink.document_id == Document.id)
+        .filter(ProjectDocumentLink.project_id == project_id, Document.archived.is_(False))
+        .order_by(ProjectDocumentLink.attached_at.desc())
+        .all()
+    )
+    return [document_to_dict(doc, link_role=link.role) for doc, link in rows]
+
+
+def list_global_documents(
+    session: Session,
+    *,
+    category: str | None = None,
+    search: str | None = None,
+    limit: int = 200,
+) -> list[dict[str, Any]]:
+    query = session.query(Document).filter(Document.scope == "global", Document.archived.is_(False))
+    if category and category.strip():
+        query = query.filter(Document.category == category.strip().lower())
+    term = (search or "").strip()
+    if term:
+        like = f"%{term}%"
+        query = query.filter(or_(Document.title.ilike(like), Document.original_filename.ilike(like)))
+    cap = max(1, min(int(limit), 500))
+    docs = query.order_by(Document.created_at.desc()).limit(cap).all()
+    return [document_to_dict(d) for d in docs]
+
+
+def promote_document_to_global(session: Session, document_id: int) -> dict[str, Any]:
+    """Promote a project-local document to the global library.
+
+    Re-scopes the row and moves its search_index rows to the global shell project
+    (embeddings preserved, no re-index). The project link stays so the doc remains
+    visible under its original project.
+    """
+    from services.global_brain_support import get_global_brain_shell_project_id
+
+    doc = (
+        session.query(Document)
+        .filter(Document.id == document_id, Document.archived.is_(False))
+        .first()
+    )
+    if not doc:
+        raise ValueError("Document not found")
+    if doc.scope == "global":
+        return {"document": document_to_dict(doc), "promoted": False, "reason": "already_global"}
+    if doc.scope != "project_local":
+        raise ValueError(f"Cannot promote document with scope {doc.scope!r}")
+
+    old_project_id = doc.owner_project_id
+    global_pid = get_global_brain_shell_project_id(session)
+
+    moved = 0
+    if old_project_id is not None:
+        result = session.execute(
+            text(
+                """
+                UPDATE search_index
+                SET project_id = :gpid
+                WHERE project_id = :old_pid
+                  AND source_type = 'document'
+                  AND source_id = :sid
+                """
+            ),
+            {"gpid": global_pid, "old_pid": old_project_id, "sid": str(doc.id)},
+        )
+        moved = result.rowcount or 0
+
+    doc.scope = "global"
+    session.commit()
+    session.refresh(doc)
+
+    return {
+        "document": document_to_dict(doc),
+        "promoted": True,
+        "previous_project_id": old_project_id,
+        "index_rows_moved": moved,
+    }
+
+
+def documents_storage_dir() -> Path:
+    """Optional local cache directory (canonical bytes live in Postgres file_content)."""
+    root = storage_root() / "documents"
+    root.mkdir(parents=True, exist_ok=True)
+    return root
+
+
+def pg_stored_path(file_hash: str, ext: str) -> str:
+    return f"pg://documents/{file_hash}{ext}"
+
+
+def read_document_bytes(document: Document) -> bytes:
+    if document.file_content:
+        return bytes(document.file_content)
+    path = Path(document.stored_path)
+    if path.is_file():
+        return path.read_bytes()
+    raise ValueError(f"Document {document.id} has no file_content and stored_path is missing on disk")
+
+
+def attach_document_to_project(
+    session: Session,
+    project_id: int,
+    document_id: int,
+    *,
+    role: str = "reference",
+) -> dict[str, Any]:
+    """Attach an existing library or project-local document to a project."""
+    proj = session.query(Project).filter(Project.id == project_id).first()
+    if not proj:
+        raise ValueError("Project not found")
+
+    doc = (
+        session.query(Document)
+        .filter(Document.id == document_id, Document.archived.is_(False))
+        .first()
+    )
+    if not doc:
+        raise ValueError("Document not found")
+
+    if doc.scope == "global":
+        pass
+    elif doc.scope == "project_local":
+        if doc.owner_project_id != project_id:
+            raise ValueError(
+                "Document belongs to another project. Promote to global library first or attach from that project."
+            )
+    else:
+        raise ValueError("Invalid document scope")
+
+    role_norm = "schematic" if (role or "").lower() == "schematic" else "reference"
+    link = (
+        session.query(ProjectDocumentLink)
+        .filter(
+            ProjectDocumentLink.project_id == project_id,
+            ProjectDocumentLink.document_id == document_id,
+        )
+        .first()
+    )
+    if link:
+        if link.role != role_norm:
+            link.role = role_norm
+            session.commit()
+            session.refresh(link)
+        return {
+            "document": document_to_dict(doc, link_role=link.role),
+            "link": {"project_id": project_id, "document_id": document_id, "role": link.role},
+            "already_linked": True,
+        }
+
+    link = ProjectDocumentLink(
+        project_id=project_id,
+        document_id=document_id,
+        role=role_norm,
+    )
+    session.add(link)
+    session.commit()
+    session.refresh(link)
+    return {
+        "document": document_to_dict(doc, link_role=link.role),
+        "link": {"project_id": project_id, "document_id": document_id, "role": link.role},
+        "already_linked": False,
+    }
+
+
+def register_document_from_path(
+    session: Session,
+    source_path: Path,
+    *,
+    scope: str,
+    project_id: int | None = None,
+    title: str | None = None,
+    category: str = "other",
+    attach: bool = True,
+) -> dict[str, Any]:
+    """Copy a file into shared/storage/documents/ and create Document (+ optional link)."""
+    if not source_path.is_file():
+        raise ValueError(f"File not found: {source_path}")
+
+    scope_norm = (scope or "project").strip().lower()
+    if scope_norm == "global":
+        doc_scope = "global"
+        owner_project_id = None
+        attach_project_id = project_id
+    elif scope_norm == "project":
+        if project_id is None:
+            raise ValueError("project_id is required when scope is project")
+        doc_scope = "project_local"
+        owner_project_id = project_id
+        attach_project_id = project_id
+        proj = session.query(Project).filter(Project.id == project_id).first()
+        if not proj:
+            raise ValueError("Project not found")
+    else:
+        raise ValueError("scope must be global or project")
+
+    ext = source_path.suffix.lower()
+    if ext not in ALLOWED_EXTENSIONS:
+        raise ValueError(f"Unsupported file type. Allowed: {', '.join(sorted(ALLOWED_EXTENSIONS))}")
+
+    size = source_path.stat().st_size
+    if size > MAX_FILE_SIZE:
+        raise ValueError(f"File exceeds {MAX_FILE_SIZE // (1024 * 1024)}MB limit")
+
+    file_hash = _file_hash(source_path)
+    existing = session.query(Document).filter(Document.file_hash == file_hash, Document.archived.is_(False)).first()
+    file_bytes = source_path.read_bytes()
+
+    if existing:
+        doc = existing
+        created = False
+        if not doc.file_content:
+            doc.file_content = file_bytes
+            session.commit()
+            session.refresh(doc)
+    else:
+        dest_name = f"{file_hash}{ext}"
+        logical_path = pg_stored_path(file_hash, ext)
+        # Optional local cache for operator inspection (not canonical).
+        cache_path = documents_storage_dir() / dest_name
+        if not cache_path.exists():
+            cache_path.write_bytes(file_bytes)
+
+        ttl = (title or "").strip() or source_path.stem
+        doc = Document(
+            scope=doc_scope,
+            owner_project_id=owner_project_id,
+            title=ttl[:255],
+            original_filename=source_path.name[:255],
+            stored_path=logical_path,
+            file_hash=file_hash,
+            mime_type=mime_for_ext(ext),
+            file_size=size,
+            category=normalize_category(category),
+            file_content=file_bytes,
+        )
+        session.add(doc)
+        session.commit()
+        session.refresh(doc)
+        created = True
+
+    linked = False
+    if attach and attach_project_id is not None:
+        result = attach_document_to_project(session, attach_project_id, doc.id)
+        linked = not result.get("already_linked", False)
+
+    return {
+        "document": document_to_dict(doc),
+        "created": created,
+        "linked": linked,
+        "scope": doc.scope,
+    }

package/scripts/services/embedding_service.py

@@ -0,0 +1,72 @@
+"""Sync Ollama embedding service for search index vectors."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+import httpx
+
+from _db_lib import ollama_base_url
+
+logger = logging.getLogger(__name__)
+
+EMBEDDING_DIM = 768
+BATCH_SIZE = 64
+DEFAULT_EMBEDDING_MODEL = "nomic-embed-text"
+
+
+def _normalize_endpoint(endpoint: str) -> str:
+    return endpoint.rstrip("/").replace("/v1", "")
+
+
+def generate_embeddings(
+    texts: list[str],
+    endpoint: str | None = None,
+    model: str | None = None,
+    batch_size: int = BATCH_SIZE,
+) -> list[list[float] | None]:
+    """Generate embeddings for texts via Ollama /api/embed (sync).
+
+    Returns a list parallel to *texts*; each element is a vector or None on failure.
+    """
+    if not texts:
+        return []
+
+    base = _normalize_endpoint(endpoint or ollama_base_url())
+    embed_model = (model or DEFAULT_EMBEDDING_MODEL).strip() or DEFAULT_EMBEDDING_MODEL
+    results: list[list[float] | None] = [None] * len(texts)
+
+    with httpx.Client(timeout=120.0) as client:
+        for start in range(0, len(texts), batch_size):
+            batch = texts[start : start + batch_size]
+            try:
+                resp = client.post(
+                    f"{base}/api/embed",
+                    json={"model": embed_model, "input": batch},
+                )
+                resp.raise_for_status()
+                data: dict[str, Any] = resp.json()
+                embeddings = data.get("embeddings") or []
+                for i, emb in enumerate(embeddings):
+                    if i < len(batch) and emb:
+                        results[start + i] = emb
+            except Exception as exc:
+                logger.error(
+                    "Embedding batch failed offset=%d size=%d: %s",
+                    start,
+                    len(batch),
+                    exc,
+                )
+
+    return results
+
+
+def generate_single_embedding(
+    text: str,
+    endpoint: str | None = None,
+    model: str | None = None,
+) -> list[float] | None:
+    """Generate one embedding vector; None on failure."""
+    results = generate_embeddings([text], endpoint=endpoint, model=model, batch_size=1)
+    return results[0] if results else None

package/scripts/services/global_brain_support.py

@@ -0,0 +1,36 @@
+"""Org-level global brain shell project (hidden Projects row for search_index FK)."""
+
+from __future__ import annotations
+
+from sqlalchemy.orm import Session
+
+from models import Project
+from services.brain_paths import GLOBAL_BRAIN_PROJECT_NAME, ensure_global_brain_layout
+
+
+def ensure_global_brain_shell_project(session: Session) -> Project:
+    """Return or create the hidden project backing global/brain/ search rows."""
+    project = session.query(Project).filter(Project.name == GLOBAL_BRAIN_PROJECT_NAME).first()
+    if project:
+        if not project.is_system_hidden:
+            project.is_system_hidden = True
+            session.commit()
+            session.refresh(project)
+        return project
+
+    ensure_global_brain_layout()
+    project = Project(
+        name=GLOBAL_BRAIN_PROJECT_NAME,
+        is_system_hidden=True,
+        l5x_status="pending",
+        org_id="default",
+    )
+    session.add(project)
+    session.commit()
+    session.refresh(project)
+    return project
+
+
+def get_global_brain_shell_project_id(session: Session) -> int:
+    """Database id for the global brain shell project."""
+    return ensure_global_brain_shell_project(session).id

package/scripts/services/maintenance_log_service.py

@@ -0,0 +1,258 @@
+"""Maintenance log CRUD, search, soft delete, and revision history."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Any
+
+from sqlalchemy import func, or_
+from sqlalchemy.orm import Session
+
+from models import MaintenanceLogEntry, MaintenanceLogRevision, Project
+
+
+def log_entry_to_dict(entry: MaintenanceLogEntry, *, project_name: str | None = None) -> dict[str, Any]:
+    return {
+        "id": entry.id,
+        "project_id": entry.project_id,
+        "project_name": project_name,
+        "title": entry.title,
+        "body": entry.body,
+        "fault_code": entry.fault_code,
+        "equipment_tag": entry.equipment_tag,
+        "logged_by": entry.logged_by,
+        "deleted_at": entry.deleted_at.isoformat() if entry.deleted_at else None,
+        "created_at": entry.created_at.isoformat() if entry.created_at else None,
+        "updated_at": entry.updated_at.isoformat() if entry.updated_at else None,
+    }
+
+
+def _active_logs_query(session: Session):
+    return session.query(MaintenanceLogEntry, Project.name).outerjoin(
+        Project, MaintenanceLogEntry.project_id == Project.id
+    ).filter(MaintenanceLogEntry.deleted_at.is_(None))
+
+
+def _next_log_revision(session: Session, entry_id: int) -> int:
+    current = (
+        session.query(func.max(MaintenanceLogRevision.revision_number))
+        .filter(MaintenanceLogRevision.log_entry_id == entry_id)
+        .scalar()
+    )
+    return int(current or 0) + 1
+
+
+def _save_log_revision(session: Session, entry: MaintenanceLogEntry) -> None:
+    rev = MaintenanceLogRevision(
+        log_entry_id=entry.id,
+        title=entry.title,
+        body=entry.body,
+        fault_code=entry.fault_code,
+        equipment_tag=entry.equipment_tag,
+        logged_by=entry.logged_by,
+        revision_number=_next_log_revision(session, entry.id),
+    )
+    session.add(rev)
+
+
+def create_log_entry(
+    session: Session,
+    *,
+    title: str,
+    body: str,
+    project_id: int | None = None,
+    fault_code: str | None = None,
+    equipment_tag: str | None = None,
+    logged_by: str | None = None,
+) -> dict[str, Any]:
+    clean_title = title.strip()
+    clean_body = body.strip()
+    if not clean_title:
+        raise ValueError("title is required")
+    if not clean_body:
+        raise ValueError("body is required")
+
+    project_name = None
+    if project_id is not None:
+        project = session.query(Project).filter(Project.id == project_id).first()
+        if not project:
+            raise ValueError(f"Project {project_id} not found")
+        project_name = project.name
+
+    entry = MaintenanceLogEntry(
+        project_id=project_id,
+        title=clean_title,
+        body=clean_body,
+        fault_code=(fault_code or "").strip() or None,
+        equipment_tag=(equipment_tag or "").strip() or None,
+        logged_by=(logged_by or "").strip() or None,
+    )
+    session.add(entry)
+    session.commit()
+    session.refresh(entry)
+    return log_entry_to_dict(entry, project_name=project_name)
+
+
+def update_log_entry(
+    session: Session,
+    entry_id: int,
+    *,
+    title: str | None = None,
+    body: str | None = None,
+    fault_code: str | None = None,
+    equipment_tag: str | None = None,
+    logged_by: str | None = None,
+) -> dict[str, Any]:
+    entry = (
+        session.query(MaintenanceLogEntry)
+        .filter(MaintenanceLogEntry.id == entry_id, MaintenanceLogEntry.deleted_at.is_(None))
+        .first()
+    )
+    if not entry:
+        raise ValueError(f"Log entry {entry_id} not found")
+
+    _save_log_revision(session, entry)
+
+    if title is not None and title.strip():
+        entry.title = title.strip()
+    if body is not None and body.strip():
+        entry.body = body.strip()
+    if fault_code is not None:
+        entry.fault_code = fault_code.strip() or None
+    if equipment_tag is not None:
+        entry.equipment_tag = equipment_tag.strip() or None
+    if logged_by is not None:
+        entry.logged_by = logged_by.strip() or None
+
+    entry.updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
+    session.commit()
+    session.refresh(entry)
+
+    project_name = None
+    if entry.project_id:
+        proj = session.query(Project).filter(Project.id == entry.project_id).first()
+        project_name = proj.name if proj else None
+    return log_entry_to_dict(entry, project_name=project_name)
+
+
+def soft_delete_log_entry(session: Session, entry_id: int) -> dict[str, Any]:
+    entry = (
+        session.query(MaintenanceLogEntry)
+        .filter(MaintenanceLogEntry.id == entry_id, MaintenanceLogEntry.deleted_at.is_(None))
+        .first()
+    )
+    if not entry:
+        raise ValueError(f"Log entry {entry_id} not found")
+
+    _save_log_revision(session, entry)
+    entry.deleted_at = datetime.now(timezone.utc).replace(tzinfo=None)
+    session.commit()
+    return {"ok": True, "id": entry_id, "soft_deleted": True}
+
+
+def restore_log_entry(session: Session, entry_id: int) -> dict[str, Any]:
+    entry = (
+        session.query(MaintenanceLogEntry)
+        .filter(MaintenanceLogEntry.id == entry_id, MaintenanceLogEntry.deleted_at.isnot(None))
+        .first()
+    )
+    if not entry:
+        raise ValueError(f"No deleted log entry {entry_id} to restore")
+
+    entry.deleted_at = None
+    session.commit()
+    session.refresh(entry)
+
+    project_name = None
+    if entry.project_id:
+        proj = session.query(Project).filter(Project.id == entry.project_id).first()
+        project_name = proj.name if proj else None
+    return log_entry_to_dict(entry, project_name=project_name)
+
+
+def list_log_revisions(session: Session, entry_id: int, *, limit: int = 20) -> list[dict[str, Any]]:
+    cap = max(1, min(int(limit), 100))
+    rows = (
+        session.query(MaintenanceLogRevision)
+        .filter(MaintenanceLogRevision.log_entry_id == entry_id)
+        .order_by(MaintenanceLogRevision.revision_number.desc())
+        .limit(cap)
+        .all()
+    )
+    return [
+        {
+            "revision_number": r.revision_number,
+            "title": r.title,
+            "body_preview": (r.body[:400] + "…") if len(r.body) > 400 else r.body,
+            "fault_code": r.fault_code,
+            "equipment_tag": r.equipment_tag,
+            "logged_by": r.logged_by,
+            "created_at": r.created_at.isoformat() if r.created_at else None,
+        }
+        for r in rows
+    ]
+
+
+def search_log_entries(
+    session: Session,
+    *,
+    query: str,
+    project_id: int | None = None,
+    fault_code: str | None = None,
+    equipment_tag: str | None = None,
+    limit: int = 25,
+    include_deleted: bool = False,
+) -> list[dict[str, Any]]:
+    term = (query or "").strip()
+    cap = max(1, min(int(limit), 100))
+
+    base = session.query(MaintenanceLogEntry, Project.name).outerjoin(
+        Project, MaintenanceLogEntry.project_id == Project.id
+    )
+    if not include_deleted:
+        base = base.filter(MaintenanceLogEntry.deleted_at.is_(None))
+
+    if project_id is not None:
+        base = base.filter(MaintenanceLogEntry.project_id == project_id)
+    if fault_code and fault_code.strip():
+        base = base.filter(MaintenanceLogEntry.fault_code.ilike(fault_code.strip()))
+    if equipment_tag and equipment_tag.strip():
+        base = base.filter(MaintenanceLogEntry.equipment_tag.ilike(f"%{equipment_tag.strip()}%"))
+
+    if term:
+        ts_query = func.plainto_tsquery("english", term)
+        ranked = base.filter(MaintenanceLogEntry.search_vector.op("@@")(ts_query)).order_by(
+            func.ts_rank_cd(MaintenanceLogEntry.search_vector, ts_query).desc(),
+            MaintenanceLogEntry.created_at.desc(),
+        )
+        rows = ranked.limit(cap).all()
+        if rows:
+            return [log_entry_to_dict(entry, project_name=name) for entry, name in rows]
+
+        like = f"%{term}%"
+        fallback = base.filter(
+            or_(
+                MaintenanceLogEntry.title.ilike(like),
+                MaintenanceLogEntry.body.ilike(like),
+                MaintenanceLogEntry.fault_code.ilike(like),
+                MaintenanceLogEntry.equipment_tag.ilike(like),
+            )
+        ).order_by(MaintenanceLogEntry.created_at.desc())
+        rows = fallback.limit(cap).all()
+        return [log_entry_to_dict(entry, project_name=name) for entry, name in rows]
+
+    rows = base.order_by(MaintenanceLogEntry.created_at.desc()).limit(cap).all()
+    return [log_entry_to_dict(entry, project_name=name) for entry, name in rows]
+
+
+def get_log_entry(session: Session, entry_id: int, *, include_deleted: bool = False) -> dict[str, Any] | None:
+    q = session.query(MaintenanceLogEntry, Project.name).outerjoin(
+        Project, MaintenanceLogEntry.project_id == Project.id
+    ).filter(MaintenanceLogEntry.id == entry_id)
+    if not include_deleted:
+        q = q.filter(MaintenanceLogEntry.deleted_at.is_(None))
+    row = q.first()
+    if not row:
+        return None
+    entry, name = row
+    return log_entry_to_dict(entry, project_name=name)