sylo-fieldbrain 0.1.0

package/scripts/services/document_catalog.py

@@ -0,0 +1,186 @@
+"""Catalog metadata for library documents (agent-read summary, not full-file ingest)."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from sqlalchemy.orm import Session
+
+from models import Document, DocumentAnalysis, DocumentPdfOutline
+from services.document_formats import normalize_category
+from services.document_ingest import _index_project_id_for_document, index_document_text
+from services.document_service import document_to_dict
+
+
+def _parse_tags(raw: str | list[str] | None) -> list[str]:
+    if raw is None:
+        return []
+    if isinstance(raw, list):
+        return [str(t).strip() for t in raw if str(t).strip()]
+    text = raw.strip()
+    if not text:
+        return []
+    if text.startswith("["):
+        try:
+            parsed = json.loads(text)
+            if isinstance(parsed, list):
+                return [str(t).strip() for t in parsed if str(t).strip()]
+        except json.JSONDecodeError:
+            pass
+    return [p.strip() for p in text.replace(";", ",").split(",") if p.strip()]
+
+
+def _normalize_outline(entries: list[dict[str, Any]] | None) -> list[dict[str, Any]]:
+    if not entries:
+        return []
+    out: list[dict[str, Any]] = []
+    for i, row in enumerate(entries):
+        if not isinstance(row, dict):
+            continue
+        title = str(row.get("title") or "").strip()
+        if not title:
+            continue
+        level = int(row.get("level") or 1)
+        page = row.get("page")
+        if page is None:
+            page = row.get("physical_pdf_page")
+        physical_page = int(page) if page is not None and str(page).isdigit() else 0
+        out.append(
+            {
+                "level": max(1, level),
+                "title": title,
+                "physical_pdf_page": physical_page,
+                "toc_page_claimed": row.get("toc_page_claimed"),
+                "verification": str(row.get("verification") or "agent"),
+                "sequence": int(row.get("sequence") if row.get("sequence") is not None else i),
+            }
+        )
+    return out
+
+
+def _build_index_text(
+    document: Document,
+    description: str,
+    tags: list[str],
+    outline: list[dict[str, Any]],
+) -> str:
+    parts = [
+        f"Title: {document.title}",
+        f"Filename: {document.original_filename}",
+        f"Category: {document.category}",
+    ]
+    if document.manufacturer:
+        parts.append(f"Manufacturer: {document.manufacturer}")
+    if document.model:
+        parts.append(f"Model: {document.model}")
+    if document.version:
+        parts.append(f"Version: {document.version}")
+    if tags:
+        parts.append(f"Tags: {', '.join(tags)}")
+    parts.append("")
+    parts.append(description.strip())
+    if outline:
+        parts.append("")
+        parts.append("Outline:")
+        for row in outline:
+            indent = "  " * (max(1, row["level"]) - 1)
+            page_note = f" (p.{row['physical_pdf_page'] + 1})" if row["physical_pdf_page"] else ""
+            parts.append(f"{indent}- {row['title']}{page_note}")
+    return "\n".join(parts).strip()
+
+
+def catalog_document(
+    session: Session,
+    document_id: int,
+    *,
+    description: str,
+    tags: str | list[str] | None = None,
+    category: str | None = None,
+    title: str | None = None,
+    manufacturer: str | None = None,
+    model: str | None = None,
+    version: str | None = None,
+    outline: list[dict[str, Any]] | None = None,
+    project_id: int | None = None,
+    embed: bool = True,
+) -> dict[str, Any]:
+    """Write catalog rows and index summary text for search (not full-file extraction)."""
+    desc = (description or "").strip()
+    if not desc:
+        raise ValueError("description is required — agent must summarize after reading the file.")
+
+    document = (
+        session.query(Document)
+        .filter(Document.id == document_id, Document.archived.is_(False))
+        .first()
+    )
+    if not document:
+        raise ValueError("Document not found")
+
+    tag_list = _parse_tags(tags)
+    outline_rows = _normalize_outline(outline)
+
+    if title and title.strip():
+        document.title = title.strip()[:255]
+    if category:
+        document.category = normalize_category(category)
+    if manufacturer is not None:
+        document.manufacturer = manufacturer.strip()[:100] or None
+    if model is not None:
+        document.model = model.strip()[:100] or None
+    if version is not None:
+        document.version = version.strip()[:50] or None
+    if tag_list:
+        document.tags_json = json.dumps(tag_list)
+
+    analysis = (
+        session.query(DocumentAnalysis)
+        .filter(DocumentAnalysis.document_id == document.id)
+        .first()
+    )
+    if analysis:
+        analysis.llm_index_description = desc
+    else:
+        session.add(
+            DocumentAnalysis(
+                document_id=document.id,
+                llm_index_description=desc,
+            )
+        )
+
+    session.query(DocumentPdfOutline).filter(DocumentPdfOutline.document_id == document.id).delete()
+    for row in outline_rows:
+        session.add(
+            DocumentPdfOutline(
+                document_id=document.id,
+                level=row["level"],
+                title=row["title"],
+                toc_page_claimed=row.get("toc_page_claimed"),
+                physical_pdf_page=row["physical_pdf_page"],
+                verification=row["verification"],
+                sequence=row["sequence"],
+            )
+        )
+
+    session.flush()
+    index_pid = _index_project_id_for_document(session, document, project_id)
+    index_text = _build_index_text(document, desc, tag_list, outline_rows)
+    index_stats = index_document_text(
+        session,
+        document,
+        index_text,
+        index_project_id=index_pid,
+    )
+
+    return {
+        "document": document_to_dict(document),
+        "catalog": {
+            "description_chars": len(desc),
+            "tags": tag_list,
+            "outline_entries": len(outline_rows),
+            "indexed_for_search": True,
+            "embeddings_written": index_stats.get("embeddings_written", 0) if embed else 0,
+        },
+        **index_stats,
+    }

package/scripts/services/document_formats.py

@@ -0,0 +1,116 @@
+"""Supported library file types and which Sylo skill reads them (agent-side)."""
+
+from __future__ import annotations
+
+# Extensions FieldBrain stores in the global/project document library (bytes in Postgres).
+ALLOWED_EXTENSIONS: frozenset[str] = frozenset(
+    {
+        ".pdf",
+        ".txt",
+        ".md",
+        ".markdown",
+        ".docx",
+        ".xlsx",
+        ".xlsm",
+        ".ods",
+        ".csv",
+        ".jpg",
+        ".jpeg",
+        ".png",
+        ".webp",
+        ".gif",
+        ".bmp",
+    }
+)
+
+MAX_FILE_SIZE = 50 * 1024 * 1024
+
+DOCUMENT_CATEGORIES: frozenset[str] = frozenset(
+    {
+        "manual",
+        "datasheet",
+        "requirements",
+        "email",
+        "howto",
+        "guide",
+        "schematic",
+        "spreadsheet",
+        "image",
+        "markdown",
+        "reference",
+        "other",
+    }
+)
+
+MIME_BY_EXT: dict[str, str] = {
+    ".pdf": "application/pdf",
+    ".txt": "text/plain",
+    ".md": "text/markdown",
+    ".markdown": "text/markdown",
+    ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    ".xlsm": "application/vnd.ms-excel.sheet.macroEnabled.12",
+    ".ods": "application/vnd.oasis.opendocument.spreadsheet",
+    ".csv": "text/csv",
+    ".jpg": "image/jpeg",
+    ".jpeg": "image/jpeg",
+    ".png": "image/png",
+    ".webp": "image/webp",
+    ".gif": "image/gif",
+    ".bmp": "image/bmp",
+}
+
+# Agent reads the file with these Sylo capabilities before calling fieldbrain_document_catalog.
+READER_HINTS: dict[str, str] = {
+    ".pdf": "sylo-pdf-reader: search_schematic_pdf (and region tools as needed)",
+    ".xlsx": "sylo-spreadsheet: read_spreadsheet",
+    ".xlsm": "sylo-spreadsheet: read_spreadsheet (formulas optional)",
+    ".ods": "sylo-spreadsheet: read_spreadsheet",
+    ".csv": "Pi read tool (plain text)",
+    ".txt": "Pi read tool",
+    ".md": "Pi read tool",
+    ".markdown": "Pi read tool",
+    ".docx": "Chat attachment + agent summary (extract headings/topics); template-docx-writer for embedded images only",
+    ".jpg": "Vision on attachment — describe what the image shows",
+    ".jpeg": "Vision on attachment — describe what the image shows",
+    ".png": "Vision on attachment — describe what the image shows",
+    ".webp": "Vision on attachment — describe what the image shows",
+    ".gif": "Vision on attachment — describe what the image shows",
+    ".bmp": "Vision on attachment — describe what the image shows",
+}
+
+IMAGE_EXTENSIONS: frozenset[str] = frozenset({".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp"})
+
+
+def mime_for_ext(ext: str) -> str | None:
+    return MIME_BY_EXT.get(ext.lower())
+
+
+def reader_hint_for_ext(ext: str) -> str | None:
+    return READER_HINTS.get(ext.lower())
+
+
+def normalize_category(raw: str | None) -> str:
+    cat = (raw or "other").strip().lower().replace("-", "").replace("_", "")
+    aliases = {
+        "howto": "howto",
+        "howtoguide": "howto",
+        "howguide": "howto",
+        "spec": "requirements",
+        "req": "requirements",
+        "datasheets": "datasheet",
+        "manuals": "manual",
+        "photo": "image",
+        "picture": "image",
+        "img": "image",
+        "emailchain": "email",
+        "mail": "email",
+    }
+    cat = aliases.get(cat, cat)
+    if cat in DOCUMENT_CATEGORIES:
+        return cat
+    return "other"
+
+
+def is_image_ext(ext: str) -> bool:
+    return ext.lower() in IMAGE_EXTENSIONS

package/scripts/services/document_ingest.py

@@ -0,0 +1,254 @@
+"""Document ingestion: extract text, chunk, index, optional embed via Ollama."""
+
+from __future__ import annotations
+
+import logging
+import re
+from pathlib import Path
+from typing import Any
+
+from sqlalchemy import text
+from sqlalchemy.orm import Session
+
+from _db_lib import check_pgvector, ollama_base_url
+from models import Document, GlobalSettings
+from services.document_service import ALLOWED_EXTENSIONS, register_document_from_path, read_document_bytes
+from services.embedding_service import generate_embeddings
+from services.global_brain_support import get_global_brain_shell_project_id
+
+logger = logging.getLogger(__name__)
+
+SOURCE_DOCUMENT = "document"
+CHUNK_TARGET_CHARS = 500
+
+
+def extract_text_from_bytes(data: bytes, ext: str) -> str:
+    """Extract plain text from PDF (pymupdf) or .txt bytes."""
+    ext_l = ext.lower()
+    if ext_l == ".pdf":
+        import fitz
+
+        doc = fitz.open(stream=data, filetype="pdf")
+        try:
+            parts = [page.get_text() for page in doc]
+        finally:
+            doc.close()
+        return "\n".join(parts).strip()
+    if ext_l == ".txt":
+        return data.decode("utf-8", errors="replace").strip()
+    raise ValueError(f"Unsupported file type {ext_l}. Allowed: {', '.join(sorted(ALLOWED_EXTENSIONS))}")
+
+
+def extract_text_from_file(path: Path) -> str:
+    """Extract plain text from PDF (pymupdf) or .txt on disk."""
+    return extract_text_from_bytes(path.read_bytes(), path.suffix.lower())
+
+
+def extract_text_from_document(document: Document) -> str:
+    ext = Path(document.original_filename or document.stored_path).suffix.lower()
+    if not ext and document.mime_type == "application/pdf":
+        ext = ".pdf"
+    data = read_document_bytes(document)
+    return extract_text_from_bytes(data, ext)
+
+
+def chunk_text_chars(text_body: str, target_chars: int = CHUNK_TARGET_CHARS) -> list[str]:
+    """Split text into ~target_chars chunks on sentence boundaries."""
+    if not text_body or not text_body.strip():
+        return []
+
+    sentences = re.split(r"(?<=[.!?])\s+", text_body.strip())
+    chunks: list[str] = []
+    current: list[str] = []
+    current_len = 0
+
+    for sent in sentences:
+        sent = sent.strip()
+        if not sent:
+            continue
+        sent_len = len(sent)
+        if current and current_len + sent_len + 1 > target_chars:
+            chunks.append(" ".join(current))
+            current = [sent]
+            current_len = sent_len
+        else:
+            current.append(sent)
+            current_len += sent_len + (1 if current_len else 0)
+
+    if current:
+        chunks.append(" ".join(current))
+
+    return chunks or [text_body.strip()]
+
+
+def _insert_index_row(
+    session: Session,
+    project_id: int,
+    *,
+    source_id: str,
+    label: str,
+    content: str,
+    chunk_index: int,
+) -> int:
+    row = session.execute(
+        text(
+            """
+            INSERT INTO search_index
+                (project_id, source_type, source_id, label, path, content, search_vector, chunk_index)
+            VALUES
+                (:pid, :stype, :sid, :label, :path, :content,
+                 setweight(to_tsvector('english', coalesce(:label,'')), 'A') ||
+                   setweight(to_tsvector('english', coalesce(:path,'')), 'B') ||
+                   setweight(to_tsvector('english', coalesce(:content,'')), 'D'),
+                 :ci)
+            RETURNING id
+            """
+        ),
+        {
+            "pid": project_id,
+            "stype": SOURCE_DOCUMENT,
+            "sid": source_id[:500],
+            "label": label[:500],
+            "path": label[:500],
+            "content": content or " ",
+            "ci": chunk_index,
+        },
+    ).fetchone()
+    return int(row[0])
+
+
+def _remove_document_index_rows(session: Session, project_id: int, document_id: int) -> None:
+    session.execute(
+        text(
+            """
+            DELETE FROM search_index
+            WHERE project_id = :pid AND source_type = :stype AND source_id = :sid
+            """
+        ),
+        {"pid": project_id, "stype": SOURCE_DOCUMENT, "sid": str(document_id)},
+    )
+
+
+def _embed_index_rows(session: Session, row_ids: list[int]) -> int:
+    if not row_ids:
+        return 0
+
+    pg_ok, _ = check_pgvector()
+    if not pg_ok:
+        return 0
+
+    settings = session.query(GlobalSettings).filter(GlobalSettings.id == 1).first()
+    model = (settings.embedding_model if settings else None) or "nomic-embed-text"
+    endpoint = ollama_base_url()
+
+    rows = session.execute(
+        text("SELECT id, content FROM search_index WHERE id = ANY(:ids)"),
+        {"ids": row_ids},
+    ).fetchall()
+    if not rows:
+        return 0
+
+    texts = [(r[1] or " ") for r in rows]
+    embeddings = generate_embeddings(texts, endpoint=endpoint, model=model)
+    updated = 0
+    for (row_id, _), emb in zip(rows, embeddings):
+        if not emb:
+            continue
+        emb_str = "[" + ",".join(str(v) for v in emb) + "]"
+        session.execute(
+            text("UPDATE search_index SET embedding = CAST(:emb AS vector) WHERE id = :id"),
+            {"emb": emb_str, "id": row_id},
+        )
+        updated += 1
+    return updated
+
+
+def _index_project_id_for_document(session: Session, doc: Document, project_id: int | None) -> int:
+    scope = (doc.scope or "project_local").strip().lower()
+    if scope == "global":
+        return get_global_brain_shell_project_id(session)
+    if project_id is not None:
+        return project_id
+    if doc.owner_project_id is not None:
+        return int(doc.owner_project_id)
+    raise ValueError("project_id required for project-scoped document indexing")
+
+
+def index_document_text(
+    session: Session,
+    document: Document,
+    full_text: str,
+    *,
+    index_project_id: int,
+) -> dict[str, Any]:
+    """Chunk document text into search_index and optionally embed."""
+    _remove_document_index_rows(session, index_project_id, document.id)
+    label = document.title or document.original_filename or "Document"
+    chunks = chunk_text_chars(full_text)
+    row_ids: list[int] = []
+
+    for ci, chunk in enumerate(chunks):
+        header = f"[{label}]"
+        body = f"{header}\n\n{chunk.strip()}"
+        row_id = _insert_index_row(
+            session,
+            index_project_id,
+            source_id=str(document.id),
+            label=label,
+            content=body,
+            chunk_index=ci,
+        )
+        row_ids.append(row_id)
+
+    embedded = _embed_index_rows(session, row_ids)
+    session.commit()
+    return {
+        "chunks_indexed": len(row_ids),
+        "embeddings_written": embedded,
+        "index_project_id": index_project_id,
+    }
+
+
+def ingest_document_file(
+    session: Session,
+    file_path: str | Path,
+    *,
+    scope: str,
+    project_id: int | None = None,
+    title: str | None = None,
+) -> dict[str, Any]:
+    """Register (if needed), extract text, index, and optionally embed a document."""
+    path = Path(file_path).expanduser().resolve()
+    reg = register_document_from_path(
+        session,
+        path,
+        scope=scope,
+        project_id=project_id,
+        title=title,
+        attach=True,
+    )
+    document = (
+        session.query(Document)
+        .filter(Document.id == reg["document"]["id"])
+        .first()
+    )
+    if not document:
+        raise ValueError("Document registration failed")
+
+    full_text = extract_text_from_document(document)
+    index_pid = _index_project_id_for_document(session, document, project_id)
+    index_stats = index_document_text(
+        session,
+        document,
+        full_text,
+        index_project_id=index_pid,
+    )
+
+    return {
+        "document_id": document.id,
+        "title": document.title,
+        "scope": document.scope,
+        "stored_path": document.stored_path,
+        "text_chars": len(full_text),
+        **index_stats,
+    }