codealmanac 0.1.0.dev0__py3-none-any.whl

codealmanac/services/viewer/requests.py

@@ -0,0 +1,86 @@
+from pathlib import Path
+
+from pydantic import field_validator
+
+from codealmanac.core.models import CodeAlmanacModel
+from codealmanac.core.text import required_text
+from codealmanac.services.wiki.paths import (
+    looks_like_dir,
+    normalize_reference_path_preserving_case,
+)
+
+
+class ViewerOverviewRequest(CodeAlmanacModel):
+    cwd: Path
+    wiki: str | None = None
+    page_limit: int = 30
+
+    @field_validator("page_limit")
+    @classmethod
+    def non_negative_page_limit(cls, value: int) -> int:
+        if value < 0:
+            raise ValueError("page_limit must be non-negative")
+        return value
+
+
+class ViewerPageRequest(CodeAlmanacModel):
+    cwd: Path
+    slug: str
+    wiki: str | None = None
+
+    @field_validator("slug")
+    @classmethod
+    def require_slug(cls, value: str) -> str:
+        return required_text(value, "slug")
+
+
+class ViewerSearchRequest(CodeAlmanacModel):
+    cwd: Path
+    wiki: str | None = None
+    query: str | None = None
+    limit: int = 50
+
+    @field_validator("limit")
+    @classmethod
+    def non_negative_limit(cls, value: int) -> int:
+        if value < 0:
+            raise ValueError("limit must be non-negative")
+        return value
+
+
+class ViewerFileRequest(CodeAlmanacModel):
+    cwd: Path
+    path: str
+    wiki: str | None = None
+    limit: int = 50
+
+    @field_validator("path")
+    @classmethod
+    def normalize_file_path(cls, value: str) -> str:
+        path = required_text(value, "file path")
+        normalized = normalize_reference_path_preserving_case(
+            path,
+            looks_like_dir(path),
+        )
+        if not normalized:
+            raise ValueError("file path must be repo-relative")
+        return normalized
+
+    @field_validator("limit")
+    @classmethod
+    def non_negative_limit(cls, value: int) -> int:
+        if value < 0:
+            raise ValueError("limit must be non-negative")
+        return value
+
+
+class ViewerTopicRequest(CodeAlmanacModel):
+    cwd: Path
+    slug: str
+    wiki: str | None = None
+    include_descendants: bool = False
+
+    @field_validator("slug")
+    @classmethod
+    def require_slug(cls, value: str) -> str:
+        return required_text(value, "topic slug")

codealmanac/services/viewer/service.py

@@ -0,0 +1,211 @@
+from pathlib import Path
+
+from codealmanac.core.errors import NotFoundError
+from codealmanac.core.slug import to_kebab_case
+from codealmanac.services.index.models import PageView, SearchPageResult
+from codealmanac.services.index.requests import SearchIndexRequest
+from codealmanac.services.index.service import IndexService
+from codealmanac.services.viewer.models import (
+    ViewerFile,
+    ViewerFileKind,
+    ViewerFileReference,
+    ViewerOverview,
+    ViewerPage,
+    ViewerPageSummary,
+    ViewerSearch,
+    ViewerTopic,
+    ViewerTopicSummary,
+    ViewerWorkspace,
+)
+from codealmanac.services.viewer.renderer import MarkdownRenderer
+from codealmanac.services.viewer.requests import (
+    ViewerFileRequest,
+    ViewerOverviewRequest,
+    ViewerPageRequest,
+    ViewerSearchRequest,
+    ViewerTopicRequest,
+)
+from codealmanac.services.wiki.paths import looks_like_dir
+from codealmanac.services.workspaces.models import Workspace
+from codealmanac.services.workspaces.requests import SelectWorkspaceRequest
+from codealmanac.services.workspaces.service import WorkspacesService
+
+
+class ViewerService:
+    def __init__(
+        self,
+        workspaces: WorkspacesService,
+        index: IndexService,
+        renderer: MarkdownRenderer,
+    ):
+        self.workspaces = workspaces
+        self.index = index
+        self.renderer = renderer
+
+    def overview(self, request: ViewerOverviewRequest) -> ViewerOverview:
+        workspace = self.select_workspace(request.cwd, request.wiki)
+        summary = self.index.summary(workspace.workspace_id)
+        pages = self.index.search(
+            workspace.workspace_id,
+            SearchIndexRequest(limit=request.page_limit),
+        )
+        topics = self.index.list_topics(workspace.workspace_id)
+        return ViewerOverview(
+            workspace=viewer_workspace(workspace),
+            page_count=summary.pages,
+            topic_count=summary.topics,
+            pages=tuple(page_summary_from_search(page) for page in pages),
+            topics=tuple(
+                ViewerTopicSummary(
+                    slug=topic.slug,
+                    title=topic.title,
+                    description=topic.description,
+                    page_count=topic.page_count,
+                )
+                for topic in topics
+            ),
+            featured_page=self.get_featured_page(workspace),
+        )
+
+    def page(self, request: ViewerPageRequest) -> ViewerPage:
+        workspace = self.select_workspace(request.cwd, request.wiki)
+        page = self.get_page_or_raise(workspace, request.slug)
+        related_pages = self.related_pages(workspace, page)
+        return ViewerPage(
+            workspace=viewer_workspace(workspace),
+            slug=page.slug,
+            title=page.title,
+            summary=page.summary,
+            topics=page.topics,
+            body=page.body,
+            html=self.renderer.render(page.body),
+            backlinks=page.wikilinks_in,
+            outgoing_links=page.wikilinks_out,
+            file_refs=tuple(
+                ViewerFileReference(path=ref.path, is_dir=ref.is_dir)
+                for ref in page.file_refs
+            ),
+            related_pages=related_pages,
+        )
+
+    def search(self, request: ViewerSearchRequest) -> ViewerSearch:
+        workspace = self.select_workspace(request.cwd, request.wiki)
+        pages = self.index.search(
+            workspace.workspace_id,
+            SearchIndexRequest(query=request.query, limit=request.limit),
+        )
+        return ViewerSearch(
+            workspace=viewer_workspace(workspace),
+            query=request.query,
+            pages=tuple(page_summary_from_search(page) for page in pages),
+        )
+
+    def file(self, request: ViewerFileRequest) -> ViewerFile:
+        workspace = self.select_workspace(request.cwd, request.wiki)
+        pages = self.index.search(
+            workspace.workspace_id,
+            SearchIndexRequest(mentions=request.path, limit=request.limit),
+        )
+        kind = (
+            ViewerFileKind.DIRECTORY
+            if looks_like_dir(request.path)
+            else ViewerFileKind.FILE
+        )
+        return ViewerFile(
+            workspace=viewer_workspace(workspace),
+            path=request.path,
+            kind=kind,
+            pages=tuple(page_summary_from_search(page) for page in pages),
+        )
+
+    def topic(self, request: ViewerTopicRequest) -> ViewerTopic:
+        workspace = self.select_workspace(request.cwd, request.wiki)
+        slug = to_kebab_case(request.slug)
+        topic = self.index.get_topic(
+            workspace.workspace_id,
+            slug,
+            request.include_descendants,
+        )
+        if topic is None:
+            raise NotFoundError("topic", request.slug)
+        pages = tuple(
+            page_summary
+            for page_slug in topic.pages
+            if (page_summary := self.page_summary(workspace, page_slug)) is not None
+        )
+        return ViewerTopic(
+            workspace=viewer_workspace(workspace),
+            slug=topic.slug,
+            title=topic.title,
+            description=topic.description,
+            parents=topic.parents,
+            children=topic.children,
+            pages=pages,
+        )
+
+    def select_workspace(self, cwd: Path, wiki: str | None) -> Workspace:
+        if wiki is None:
+            return self.workspaces.resolve(cwd)
+        return self.workspaces.select(
+            SelectWorkspaceRequest(selector=wiki, base_path=cwd)
+        )
+
+    def get_page_or_raise(self, workspace: Workspace, slug: str) -> PageView:
+        normalized = to_kebab_case(slug)
+        page = self.index.get_page(workspace.workspace_id, normalized)
+        if page is None:
+            raise NotFoundError("page", slug)
+        return page
+
+    def page_summary(
+        self,
+        workspace: Workspace,
+        slug: str,
+    ) -> ViewerPageSummary | None:
+        page = self.index.get_page(workspace.workspace_id, slug)
+        if page is None:
+            return None
+        return page_summary_from_view(page)
+
+    def get_featured_page(self, workspace: Workspace) -> ViewerPageSummary | None:
+        return self.page_summary(workspace, "getting-started")
+
+    def related_pages(
+        self,
+        workspace: Workspace,
+        page: PageView,
+    ) -> tuple[ViewerPageSummary, ...]:
+        seen: set[str] = set()
+        related: list[ViewerPageSummary] = []
+        for slug in (*page.wikilinks_in, *page.wikilinks_out):
+            if slug in seen or slug == page.slug:
+                continue
+            seen.add(slug)
+            summary = self.page_summary(workspace, slug)
+            if summary is not None:
+                related.append(summary)
+        return tuple(related)
+
+
+def viewer_workspace(workspace: Workspace) -> ViewerWorkspace:
+    return ViewerWorkspace(name=workspace.name, root_path=workspace.root_path)
+
+
+def page_summary_from_search(page: SearchPageResult) -> ViewerPageSummary:
+    return ViewerPageSummary(
+        slug=page.slug,
+        title=page.title,
+        summary=page.summary,
+        topics=page.topics,
+        archived=page.archived_at is not None,
+    )
+
+
+def page_summary_from_view(page: PageView) -> ViewerPageSummary:
+    return ViewerPageSummary(
+        slug=page.slug,
+        title=page.title,
+        summary=page.summary,
+        topics=page.topics,
+        archived=page.archived_at is not None,
+    )

codealmanac/services/wiki/__init__.py

@@ -0,0 +1 @@
+

codealmanac/services/wiki/documents.py

@@ -0,0 +1,83 @@
+from hashlib import sha256
+from pathlib import Path
+
+from codealmanac.core.slug import to_kebab_case
+from codealmanac.services.wiki.frontmatter import first_h1, parse_frontmatter
+from codealmanac.services.wiki.models import (
+    CrossWikiLink,
+    FileLink,
+    FileReference,
+    FolderLink,
+    PageDocument,
+    PageLink,
+)
+from codealmanac.services.wiki.paths import (
+    looks_like_dir,
+    normalize_reference_path,
+    normalize_reference_path_preserving_case,
+)
+from codealmanac.services.wiki.wikilinks import extract_wikilinks
+
+
+def load_page_document(page_path: Path, pages_path: Path) -> PageDocument | None:
+    raw = page_path.read_text(encoding="utf-8")
+    frontmatter = parse_frontmatter(raw)
+    relative_path = page_path.relative_to(pages_path).as_posix()
+    slug_source = frontmatter.page_id or page_path.stem
+    slug = to_kebab_case(slug_source)
+    if not slug:
+        return None
+
+    title = frontmatter.title or first_h1(frontmatter.body) or page_path.stem
+    file_refs = list(frontmatter_file_refs(frontmatter.files))
+    page_links: list[str] = []
+    cross_wiki_links: list[tuple[str, str]] = []
+
+    for link in extract_wikilinks(frontmatter.body):
+        if isinstance(link, PageLink):
+            page_links.append(link.target)
+        elif isinstance(link, FileLink | FolderLink):
+            file_refs.append(link.ref)
+        elif isinstance(link, CrossWikiLink):
+            cross_wiki_links.append((link.wiki, link.target))
+
+    return PageDocument(
+        slug=slug,
+        title=title,
+        summary=frontmatter.summary,
+        file_path=page_path,
+        relative_path=relative_path,
+        content_hash=sha256(raw.encode("utf-8")).hexdigest(),
+        updated_at=int(page_path.stat().st_mtime),
+        archived_at=frontmatter.archived_at,
+        superseded_by=frontmatter.superseded_by,
+        topics=tuple(to_kebab_case(topic) for topic in frontmatter.topics),
+        file_refs=dedupe_file_refs(file_refs),
+        page_links=tuple(sorted(set(page_links))),
+        cross_wiki_links=tuple(sorted(set(cross_wiki_links))),
+        body=frontmatter.body,
+    )
+
+
+def frontmatter_file_refs(files: tuple[str, ...]) -> tuple[FileReference, ...]:
+    refs: list[FileReference] = []
+    for raw in files:
+        is_dir = looks_like_dir(raw)
+        normalized = normalize_reference_path(raw, is_dir)
+        original = normalize_reference_path_preserving_case(raw, is_dir)
+        if normalized:
+            refs.append(
+                FileReference(
+                    path=normalized,
+                    original_path=original,
+                    is_dir=is_dir,
+                )
+            )
+    return tuple(refs)
+
+
+def dedupe_file_refs(refs: list[FileReference]) -> tuple[FileReference, ...]:
+    unique: dict[tuple[str, bool], FileReference] = {}
+    for ref in refs:
+        unique[(ref.path, ref.is_dir)] = ref
+    return tuple(sorted(unique.values(), key=lambda ref: (ref.path, ref.is_dir)))

codealmanac/services/wiki/frontmatter.py

@@ -0,0 +1,94 @@
+import re
+from datetime import UTC, date, datetime
+from typing import Any
+
+import frontmatter
+from pydantic import BaseModel, ConfigDict, ValidationError, field_validator
+from yaml import YAMLError
+
+from codealmanac.services.wiki.models import ParsedFrontmatter
+
+
+def parse_frontmatter(raw: str) -> ParsedFrontmatter:
+    try:
+        post = frontmatter.loads(raw)
+        fields = FrontmatterFields.model_validate(post.metadata)
+    except (YAMLError, ValueError, ValidationError):
+        return ParsedFrontmatter(body=raw)
+    return ParsedFrontmatter(
+        page_id=fields.page_id,
+        title=fields.title,
+        summary=fields.summary,
+        topics=fields.topics,
+        files=fields.files,
+        archived_at=fields.archived_at,
+        superseded_by=fields.superseded_by,
+        body=post.content,
+    )
+
+
+def strip_frontmatter(raw: str) -> str:
+    return parse_frontmatter(raw).body
+
+
+def first_h1(body: str) -> str | None:
+    for line in body.splitlines()[:40]:
+        match = re.match(r"^#\s+(.+?)\s*#*\s*$", line)
+        if match is not None:
+            return match.group(1)
+    return None
+
+
+class FrontmatterFields(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    page_id: str | None = None
+    title: str | None = None
+    summary: str | None = None
+    topics: tuple[str, ...] = ()
+    files: tuple[str, ...] = ()
+    archived_at: int | None = None
+    superseded_by: str | None = None
+
+    @field_validator("page_id", "title", "summary", "superseded_by", mode="before")
+    @classmethod
+    def optional_text(cls, value: Any) -> str | None:
+        if isinstance(value, str) and value.strip():
+            return value.strip()
+        return None
+
+    @field_validator("topics", "files", mode="before")
+    @classmethod
+    def text_tuple(cls, value: Any) -> tuple[str, ...]:
+        if not isinstance(value, list | tuple):
+            return ()
+        values: list[str] = []
+        for item in value:
+            if isinstance(item, str) and item.strip():
+                values.append(item.strip())
+        return tuple(values)
+
+    @field_validator("archived_at", mode="before")
+    @classmethod
+    def epoch_seconds(cls, value: Any) -> int | None:
+        if isinstance(value, datetime):
+            return timestamp_seconds(value)
+        if isinstance(value, date):
+            return timestamp_seconds(
+                datetime(value.year, value.month, value.day, tzinfo=UTC)
+            )
+        if isinstance(value, int | float):
+            return int(value)
+        if isinstance(value, str) and value.strip():
+            try:
+                parsed = datetime.fromisoformat(value.strip())
+            except ValueError:
+                return None
+            return timestamp_seconds(parsed)
+        return None
+
+
+def timestamp_seconds(value: datetime) -> int:
+    if value.tzinfo is None:
+        value = value.replace(tzinfo=UTC)
+    return int(value.timestamp())

codealmanac/services/wiki/frontmatter_rewrite.py

@@ -0,0 +1,142 @@
+from collections.abc import Callable
+from io import StringIO
+from pathlib import Path
+from uuid import uuid4
+
+from ruamel.yaml import YAML
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
+
+from codealmanac.core.errors import ValidationFailed
+from codealmanac.core.models import CodeAlmanacModel
+from codealmanac.core.slug import to_kebab_case
+
+
+class PageTopicsRewrite(CodeAlmanacModel):
+    path: Path
+    topics: tuple[str, ...]
+
+
+def rewrite_page_topics(path: Path, topics: tuple[str, ...]) -> None:
+    raw = path.read_bytes().decode("utf-8")
+    split = split_frontmatter(raw)
+    line_ending = "\r\n" if "\r\n" in split.frontmatter else "\n"
+    yaml = YAML(typ="rt")
+    yaml.preserve_quotes = True
+    if split.frontmatter.strip():
+        data = yaml.load(split.frontmatter) or CommentedMap()
+        if not isinstance(data, CommentedMap):
+            raise ValidationFailed(f"frontmatter must be a YAML mapping: {path}")
+    else:
+        data = CommentedMap()
+
+    apply_topics(data, topics)
+
+    output = StringIO()
+    yaml.dump(data, output)
+    frontmatter = output.getvalue().rstrip("\n")
+    if line_ending != "\n":
+        frontmatter = frontmatter.replace("\n", line_ending)
+    next_raw = f"---{line_ending}{frontmatter}{line_ending}---{line_ending}"
+    next_raw += split.body
+    temporary = path.with_name(f".{path.name}.{uuid4().hex}.tmp")
+    temporary.write_bytes(next_raw.encode("utf-8"))
+    temporary.replace(path)
+
+
+def plan_page_topic_rewrites(
+    pages_path: Path,
+    transform: Callable[[tuple[str, ...]], tuple[str, ...]],
+) -> tuple[PageTopicsRewrite, ...]:
+    if not pages_path.is_dir():
+        return ()
+    rewrites: list[PageTopicsRewrite] = []
+    for page_path in sorted(pages_path.rglob("*.md")):
+        before = read_page_topics(page_path)
+        after = canonical_topic_tuple(transform(before))
+        if after != before:
+            rewrites.append(PageTopicsRewrite(path=page_path, topics=after))
+    return tuple(rewrites)
+
+
+def apply_page_topic_rewrites(rewrites: tuple[PageTopicsRewrite, ...]) -> int:
+    for rewrite in rewrites:
+        rewrite_page_topics(rewrite.path, rewrite.topics)
+    return len(rewrites)
+
+
+def read_page_topics(path: Path) -> tuple[str, ...]:
+    raw = path.read_bytes().decode("utf-8")
+    split = split_frontmatter(raw)
+    if not split.frontmatter.strip():
+        return ()
+    yaml = YAML(typ="rt")
+    yaml.preserve_quotes = True
+    try:
+        data = yaml.load(split.frontmatter) or CommentedMap()
+    except Exception as error:
+        raise ValidationFailed(f"invalid frontmatter: {path}") from error
+    if not isinstance(data, CommentedMap):
+        raise ValidationFailed(f"frontmatter must be a YAML mapping: {path}")
+    return canonical_topic_tuple(tuple(str(item) for item in topic_sequence(data)))
+
+
+class FrontmatterSplit:
+    def __init__(self, frontmatter: str, body: str):
+        self.frontmatter = frontmatter
+        self.body = body
+
+
+def split_frontmatter(raw: str) -> FrontmatterSplit:
+    if raw.startswith("---\r\n"):
+        return split_with_delimiter(raw, "\r\n")
+    if raw.startswith("---\n"):
+        return split_with_delimiter(raw, "\n")
+    return FrontmatterSplit(frontmatter="", body=raw)
+
+
+def split_with_delimiter(raw: str, line_ending: str) -> FrontmatterSplit:
+    opener = f"---{line_ending}"
+    closer = f"{line_ending}---"
+    end = raw.find(closer, len(opener))
+    if end == -1:
+        return FrontmatterSplit(frontmatter="", body=raw)
+    frontmatter = raw[len(opener) : end]
+    body_start = end + len(closer)
+    if raw.startswith(line_ending, body_start):
+        body_start += len(line_ending)
+    body = raw[body_start:]
+    return FrontmatterSplit(frontmatter=frontmatter, body=body)
+
+
+def apply_topics(data: CommentedMap, topics: tuple[str, ...]) -> None:
+    sequence = topic_sequence(data)
+    desired = list(topics)
+    desired_set = set(desired)
+    seen: set[str] = set()
+    for index in range(len(sequence) - 1, -1, -1):
+        raw_topic = str(sequence[index])
+        if raw_topic not in desired_set or raw_topic in seen:
+            del sequence[index]
+            continue
+        sequence[index] = raw_topic
+        seen.add(raw_topic)
+    for topic in desired:
+        if topic not in seen:
+            sequence.append(topic)
+            seen.add(topic)
+    data["topics"] = sequence
+
+
+def topic_sequence(data: CommentedMap) -> CommentedSeq:
+    existing = data.get("topics")
+    if isinstance(existing, CommentedSeq):
+        return existing
+    sequence = CommentedSeq()
+    if isinstance(existing, list):
+        sequence.extend(str(item) for item in existing)
+    return sequence
+
+
+def canonical_topic_tuple(topics: tuple[str, ...]) -> tuple[str, ...]:
+    canonical = tuple(to_kebab_case(str(topic)) for topic in topics)
+    return tuple(topic for topic in dict.fromkeys(canonical) if topic)

codealmanac/services/wiki/models.py

@@ -0,0 +1,69 @@
+from enum import StrEnum
+from pathlib import Path
+
+from codealmanac.core.models import CodeAlmanacModel
+
+
+class WikilinkKind(StrEnum):
+    PAGE = "page"
+    FILE = "file"
+    FOLDER = "folder"
+    CROSS_WIKI = "xwiki"
+
+
+class FileReference(CodeAlmanacModel):
+    path: str
+    original_path: str
+    is_dir: bool
+
+
+class PageLink(CodeAlmanacModel):
+    kind: WikilinkKind
+    target: str
+
+
+class FileLink(CodeAlmanacModel):
+    kind: WikilinkKind
+    ref: FileReference
+
+
+class FolderLink(CodeAlmanacModel):
+    kind: WikilinkKind
+    ref: FileReference
+
+
+class CrossWikiLink(CodeAlmanacModel):
+    kind: WikilinkKind
+    wiki: str
+    target: str
+
+
+Wikilink = PageLink | FileLink | FolderLink | CrossWikiLink
+
+
+class ParsedFrontmatter(CodeAlmanacModel):
+    page_id: str | None = None
+    title: str | None = None
+    summary: str | None = None
+    topics: tuple[str, ...] = ()
+    files: tuple[str, ...] = ()
+    archived_at: int | None = None
+    superseded_by: str | None = None
+    body: str
+
+
+class PageDocument(CodeAlmanacModel):
+    slug: str
+    title: str
+    summary: str | None
+    file_path: Path
+    relative_path: str
+    content_hash: str
+    updated_at: int
+    archived_at: int | None
+    superseded_by: str | None
+    topics: tuple[str, ...]
+    file_refs: tuple[FileReference, ...]
+    page_links: tuple[str, ...]
+    cross_wiki_links: tuple[tuple[str, str], ...]
+    body: str