donna 0.2.0__py3-none-any.whl

donna/world/markdown.py

@@ -0,0 +1,267 @@
+import enum
+from typing import Any
+
+import pydantic
+from markdown_it import MarkdownIt
+from markdown_it.token import Token
+from markdown_it.tree import SyntaxTreeNode
+from mdformat.renderer import MDRenderer
+
+from donna.core.entities import BaseEntity
+from donna.core.errors import ErrorsList
+from donna.core.result import Err, Ok, Result, unwrap_to_error
+from donna.domain.ids import FullArtifactId
+from donna.world import errors as world_errors
+
+
+class SectionLevel(str, enum.Enum):
+    h1 = "h1"
+    h2 = "h2"
+
+
+class CodeSource(BaseEntity):
+    format: str
+    properties: dict[str, str | bool]
+    content: str
+
+    def structured_data(self) -> Result[Any, ErrorsList]:
+        if "script" in self.properties:
+            return Ok({})
+
+        if self.format == "json":
+            import json
+
+            return Ok(json.loads(self.content))
+
+        if self.format == "yaml" or self.format == "yml":
+            import yaml
+
+            return Ok(yaml.safe_load(self.content))
+
+        if self.format == "toml":
+            import tomllib
+
+            return Ok(tomllib.loads(self.content))
+
+        return Err([world_errors.MarkdownUnsupportedCodeFormat(format=self.format)])
+
+
+class SectionSource(BaseEntity):
+    level: SectionLevel
+    title: str | None
+    configs: list[CodeSource]
+
+    original_tokens: list[Token]
+    analysis_tokens: list[Token]
+
+    model_config = pydantic.ConfigDict(frozen=False)
+
+    def _as_markdown(self, tokens: list[Token], with_title: bool) -> str:
+        parts = []
+
+        if with_title and self.title is not None:
+            match self.level:
+                case SectionLevel.h1:
+                    prefix = "#"
+                case SectionLevel.h2:
+                    prefix = "##"
+
+            parts.append(f"{prefix} {self.title}")
+
+        parts.append(render_back(tokens))
+
+        return "\n".join(parts)
+
+    def as_original_markdown(self, with_title: bool) -> str:
+        return self._as_markdown(self.original_tokens, with_title)
+
+    def as_analysis_markdown(self, with_title: bool) -> str:
+        return self._as_markdown(self.analysis_tokens, with_title)
+
+    def merged_configs(self) -> Result[dict[str, Any], ErrorsList]:
+        result: dict[str, Any] = {}
+        errors: ErrorsList = []
+
+        for config in self.configs:
+            data_result = config.structured_data()
+            if data_result.is_err():
+                errors.extend(data_result.unwrap_err())
+                continue
+            result.update(data_result.unwrap())
+
+        if errors:
+            return Err(errors)
+
+        return Ok(result)
+
+    def scripts(self) -> list[str]:
+        return [config.content for config in self.configs if "script" in config.properties]
+
+
+def render_back(tokens: list[Token]) -> str:
+    renderer = MDRenderer()
+    return renderer.render(tokens, {}, {})
+
+
+def clear_heading(text: str) -> str:
+    return text.lstrip("#").strip()
+
+
+def _parse_h1(
+    sections: list[SectionSource], node: SyntaxTreeNode, artifact_id: FullArtifactId | None
+) -> Result[SyntaxTreeNode | None, ErrorsList]:
+    section = sections[-1]
+
+    if section.level != SectionLevel.h1:
+        return Err([world_errors.MarkdownMultipleH1Sections(artifact_id=artifact_id)])
+
+    if section.title is not None:
+        return Err([world_errors.MarkdownMultipleH1Titles(artifact_id=artifact_id)])
+
+    section.title = clear_heading(render_back(node.to_tokens()).strip())
+
+    return Ok(node.next_sibling)
+
+
+def _parse_h2(
+    sections: list[SectionSource], node: SyntaxTreeNode, artifact_id: FullArtifactId | None
+) -> Result[SyntaxTreeNode | None, ErrorsList]:
+    section = sections[-1]
+
+    if section.title is None:
+        return Err([world_errors.MarkdownH2BeforeH1Title(artifact_id=artifact_id)])
+
+    new_section = SectionSource(
+        level=SectionLevel.h2,
+        title=clear_heading(render_back(node.to_tokens()).strip()),
+        original_tokens=[],
+        analysis_tokens=[],
+        configs=[],
+    )
+
+    sections.append(new_section)
+
+    return Ok(node.next_sibling)
+
+
+def _parse_heading(
+    sections: list[SectionSource], node: SyntaxTreeNode, artifact_id: FullArtifactId | None
+) -> Result[SyntaxTreeNode | None, ErrorsList]:
+    section = sections[-1]
+
+    if node.tag == "h1":
+        return _parse_h1(sections, node, artifact_id)
+
+    if node.tag == "h2":
+        return _parse_h2(sections, node, artifact_id)
+
+    section.original_tokens.extend(node.to_tokens())
+    return Ok(node.next_sibling)
+
+
+def _parse_fence(sections: list[SectionSource], node: SyntaxTreeNode) -> SyntaxTreeNode | None:  # noqa: CCR001
+    section = sections[-1]
+
+    info_parts = node.info.split()
+
+    format = info_parts[0] if info_parts else ""
+
+    properties: dict[str, str | bool] = {}
+
+    for part in info_parts[1:]:
+        if "=" in part:
+            key, value = part.split("=", 1)
+            properties[key] = value
+            continue
+
+        properties[part] = True
+
+    if "donna" in properties:
+        code_block = CodeSource(
+            format=format,
+            properties=properties,
+            content=node.content,
+        )
+
+        section.configs.append(code_block)
+    else:
+        section.original_tokens.extend(node.to_tokens())
+
+    return node.next_sibling
+
+
+def _parse_nested(sections: list[SectionSource], node: SyntaxTreeNode) -> SyntaxTreeNode | None:
+    section = sections[-1]
+
+    assert node.nester_tokens is not None
+
+    section.original_tokens.append(node.nester_tokens.opening)
+
+    return node.children[0]
+
+
+def _parse_others(sections: list[SectionSource], node: SyntaxTreeNode) -> SyntaxTreeNode | None:
+    section = sections[-1]
+
+    section.original_tokens.extend(node.to_tokens())
+
+    current: SyntaxTreeNode | None = node
+
+    while current is not None and current.type != "root" and current.next_sibling is None:
+        current = current.parent
+
+        if current is None:
+            break
+
+        if current.type != "root":
+            assert current.nester_tokens is not None
+            section.original_tokens.append(current.nester_tokens.closing)
+
+    return current
+
+
+@unwrap_to_error
+def parse(  # noqa: CCR001, CFQ001
+    text: str, *, artifact_id: FullArtifactId | None = None
+) -> Result[list[SectionSource], ErrorsList]:  # pylint: disable=R0912, R0915
+    md = MarkdownIt("commonmark")  # TODO: later we may want to customize it with plugins
+
+    tokens = md.parse(text)
+
+    # we do not need root node
+    node: SyntaxTreeNode | None = SyntaxTreeNode(tokens).children[0]
+
+    sections: list[SectionSource] = [
+        SectionSource(
+            level=SectionLevel.h1,
+            title=None,
+            original_tokens=[],
+            analysis_tokens=[],
+            configs=[],
+        )
+    ]
+
+    while node is not None:
+
+        if node.type == "heading":
+            node = _parse_heading(sections, node, artifact_id).unwrap()
+            continue
+
+        if node.type == "fence":
+            node = _parse_fence(sections, node)
+            continue
+
+        if node.is_nested:
+            node = _parse_nested(sections, node)
+            continue
+
+        node = _parse_others(sections, node)
+
+        if node is None:
+            break
+
+        node = node.next_sibling
+
+    return Ok(sections)
+
+    return sections

donna/world/sources/__init__.py

@@ -0,0 +1 @@
+"""Helpers for constructing artifacts from different source types."""

donna/world/sources/base.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING
+
+import pydantic
+
+from donna.core.entities import BaseEntity
+from donna.core.errors import ErrorsList
+from donna.core.result import Result
+from donna.machine.primitives import Primitive
+
+if TYPE_CHECKING:
+    from donna.domain.ids import FullArtifactId
+    from donna.machine.artifacts import Artifact
+    from donna.world.artifacts import ArtifactRenderContext
+    from donna.world.config import SourceConfig as SourceConfigModel
+
+
+class SourceConfig(BaseEntity, ABC):
+    kind: str
+    supported_extensions: list[str] = pydantic.Field(default_factory=list)
+
+    @classmethod
+    def normalize_extension(cls, extension: str) -> str:
+        normalized = str(extension).strip().lower()
+
+        if not normalized:
+            raise ValueError("Extension must not be empty")
+
+        if not normalized.startswith("."):
+            normalized = f".{normalized}"
+
+        if normalized == ".":
+            raise ValueError("Extension must include characters after '.'")
+
+        return normalized
+
+    @pydantic.field_validator("supported_extensions")
+    @classmethod
+    def _normalize_supported_extensions(cls, values: list[str]) -> list[str]:
+        normalized: list[str] = []
+
+        for value in values:
+            extension = cls.normalize_extension(value)
+            if extension not in normalized:
+                normalized.append(extension)
+
+        return normalized
+
+    def supports_extension(self, extension: str) -> bool:
+        return self.normalize_extension(extension) in self.supported_extensions
+
+    @abstractmethod
+    def construct_artifact_from_bytes(  # noqa: E704
+        self, full_id: "FullArtifactId", content: bytes, render_context: "ArtifactRenderContext"
+    ) -> Result["Artifact", ErrorsList]: ...  # noqa: E704
+
+
+class SourceConstructor(Primitive, ABC):
+    @abstractmethod
+    def construct_source(self, config: SourceConfigModel) -> SourceConfig: ...  # noqa: E704

donna/world/sources/markdown.py

@@ -0,0 +1,260 @@
+import uuid
+from typing import TYPE_CHECKING, Any, ClassVar, Literal, Protocol, cast
+
+from donna.core.errors import ErrorsList
+from donna.core.result import Err, Ok, Result, unwrap_to_error
+from donna.domain.ids import ArtifactSectionId, FullArtifactId, PythonImportPath
+from donna.machine.artifacts import Artifact, ArtifactSection, ArtifactSectionConfig, ArtifactSectionMeta
+from donna.machine.primitives import Primitive, resolve_primitive
+from donna.world import errors as world_errors
+from donna.world import markdown
+from donna.world.artifacts import ArtifactRenderContext
+from donna.world.sources.base import SourceConfig, SourceConstructor
+from donna.world.templates import RenderMode, render
+
+
+class MarkdownSectionConstructor(Protocol):
+    def markdown_construct_section(
+        self,
+        artifact_id: FullArtifactId,
+        source: markdown.SectionSource,
+        config: dict[str, Any],
+        primary: bool = False,
+    ) -> Result[ArtifactSection, ErrorsList]:
+        pass
+
+
+class Config(SourceConfig):
+    kind: Literal["markdown"] = "markdown"
+    supported_extensions: list[str] = [".md", ".markdown"]
+    default_section_kind: PythonImportPath = PythonImportPath("donna.lib.text")
+    default_primary_section_id: ArtifactSectionId = ArtifactSectionId("primary")
+
+    def construct_artifact_from_bytes(
+        self, full_id: FullArtifactId, content: bytes, render_context: ArtifactRenderContext
+    ) -> Result[Artifact, ErrorsList]:
+        return construct_artifact_from_bytes(full_id, content, render_context, self)
+
+
+class MarkdownSourceConstructor(SourceConstructor):
+    def construct_source(self, config: "SourceConfigModel") -> Config:
+        data = config.model_dump()
+        data.pop("kind", None)
+        return Config.model_validate(data)
+
+
+class MarkdownSectionMixin:
+    config_class: ClassVar[type[ArtifactSectionConfig]]
+
+    def markdown_build_title(
+        self,
+        artifact_id: FullArtifactId,
+        source: markdown.SectionSource,
+        section_config: ArtifactSectionConfig,
+        primary: bool = False,
+    ) -> str:
+        return source.title or ""
+
+    def markdown_build_description(
+        self,
+        artifact_id: FullArtifactId,
+        source: markdown.SectionSource,
+        section_config: ArtifactSectionConfig,
+        primary: bool = False,
+    ) -> str:
+        return source.as_original_markdown(with_title=False)
+
+    def markdown_construct_meta(
+        self,
+        artifact_id: FullArtifactId,
+        source: markdown.SectionSource,
+        section_config: ArtifactSectionConfig,
+        description: str,
+        primary: bool = False,
+    ) -> Result[ArtifactSectionMeta, ErrorsList]:
+        return Ok(ArtifactSectionMeta())
+
+    @unwrap_to_error
+    def markdown_construct_section(  # noqa: CCR001
+        self,
+        artifact_id: FullArtifactId,
+        source: markdown.SectionSource,
+        config: dict[str, Any],
+        primary: bool = False,
+    ) -> Result[ArtifactSection, ErrorsList]:
+        section_config = self.config_class.parse_obj(config)
+
+        title = self.markdown_build_title(
+            artifact_id=artifact_id,
+            source=source,
+            section_config=section_config,
+            primary=primary,
+        )
+        description = self.markdown_build_description(
+            artifact_id=artifact_id,
+            source=source,
+            section_config=section_config,
+            primary=primary,
+        )
+        meta = self.markdown_construct_meta(
+            artifact_id=artifact_id,
+            source=source,
+            section_config=section_config,
+            description=description,
+            primary=primary,
+        ).unwrap()
+
+        return Ok(
+            ArtifactSection(
+                id=section_config.id,
+                artifact_id=artifact_id,
+                kind=section_config.kind,
+                title=title,
+                description=description,
+                primary=primary,
+                meta=meta,
+            )
+        )
+
+
+@unwrap_to_error
+def parse_artifact_content(
+    full_id: FullArtifactId, text: str, render_context: ArtifactRenderContext
+) -> Result[list[markdown.SectionSource], ErrorsList]:
+    # Parsing an artifact two times is not ideal, but it is straightforward approach that works for now.
+    # We should consider optimizing this in the future if performance or stability becomes an issue.
+    # For now let's wait till we have more artifact analysis logic and till more use cases emerge.
+
+    original_markdown_source = render(full_id, text, render_context).unwrap()
+    original_sections = markdown.parse(original_markdown_source, artifact_id=full_id).unwrap()
+
+    analysis_context = render_context.replace(primary_mode=RenderMode.analysis)
+    analyzed_markdown_source = render(full_id, text, analysis_context).unwrap()
+    analyzed_sections = markdown.parse(analyzed_markdown_source, artifact_id=full_id).unwrap()
+
+    if len(original_sections) != len(analyzed_sections):
+        raise world_errors.MarkdownSectionsCountMismatch(
+            artifact_id=full_id,
+            original_count=len(original_sections),
+            analyzed_count=len(analyzed_sections),
+        )
+
+    if not original_sections:
+        # return Envrironment errors
+        return Err([world_errors.MarkdownArtifactWithoutSections(artifact_id=full_id)])
+
+    for original, analyzed in zip(original_sections, analyzed_sections):
+        original.analysis_tokens.extend(analyzed.original_tokens)
+
+    return Ok(original_sections)
+
+
+def construct_artifact_from_bytes(
+    full_id: FullArtifactId, content: bytes, render_context: ArtifactRenderContext, config: Config
+) -> Result[Artifact, ErrorsList]:
+    return construct_artifact_from_markdown_source(full_id, content.decode("utf-8"), render_context, config)
+
+
+@unwrap_to_error
+def construct_artifact_from_markdown_source(  # noqa: CCR001
+    full_id: FullArtifactId, content: str, render_context: ArtifactRenderContext, config: Config
+) -> Result[Artifact, ErrorsList]:
+    original_sections = parse_artifact_content(full_id, content, render_context).unwrap()
+    head_config = dict(original_sections[0].merged_configs().unwrap())
+    head_kind_value = head_config["kind"]
+    if isinstance(head_kind_value, PythonImportPath):
+        head_kind = head_kind_value
+    else:
+        head_kind = PythonImportPath.parse(head_kind_value).unwrap()
+
+    if "id" not in head_config or head_config["id"] is None:
+        head_config["id"] = config.default_primary_section_id
+
+    primary_primitive = resolve_primitive(head_kind).unwrap()
+    _ensure_markdown_constructible(primary_primitive, head_kind).unwrap()
+    markdown_primary_primitive = cast(MarkdownSectionMixin, primary_primitive)
+
+    primary_section_result = markdown_primary_primitive.markdown_construct_section(
+        artifact_id=full_id,
+        source=original_sections[0],
+        config=head_config,
+        primary=True,
+    )
+    if primary_section_result.is_err():
+        return Err(primary_section_result.unwrap_err())
+    primary_section = primary_section_result.unwrap()
+
+    sections = construct_sections_from_markdown(
+        artifact_id=full_id,
+        sections=original_sections[1:],
+        default_section_kind=config.default_section_kind,
+    ).unwrap()
+    sections = [primary_section, *sections]
+    return Ok(Artifact(id=full_id, sections=sections))
+
+
+@unwrap_to_error
+def construct_sections_from_markdown(  # noqa: CCR001
+    artifact_id: FullArtifactId,
+    sections: list[markdown.SectionSource],
+    default_section_kind: PythonImportPath,
+    primitive_overrides: dict[PythonImportPath, Primitive] | None = None,
+) -> Result[list[ArtifactSection], ErrorsList]:
+    constructed: list[ArtifactSection] = []
+    errors: ErrorsList = []
+
+    for section in sections:
+        data = dict(section.merged_configs().unwrap())
+
+        if "id" not in data or data["id"] is None:
+            data["id"] = ArtifactSectionId("markdown" + uuid.uuid4().hex.replace("-", ""))
+
+        if "kind" not in data:
+            data["kind"] = default_section_kind
+
+        kind_value = data["kind"]
+        if isinstance(kind_value, str):
+            primitive_id = PythonImportPath.parse(kind_value).unwrap()
+        else:
+            primitive_id = kind_value
+
+        primitive = _resolve_primitive(primitive_id, primitive_overrides).unwrap()
+        _ensure_markdown_constructible(primitive, primitive_id).unwrap()
+        markdown_primitive = cast(MarkdownSectionMixin, primitive)
+
+        section_result = markdown_primitive.markdown_construct_section(artifact_id, section, data, primary=False)
+        if section_result.is_err():
+            errors.extend(section_result.unwrap_err())
+            continue
+        constructed.append(section_result.unwrap())
+
+    if errors:
+        return Err(errors)
+
+    return Ok(constructed)
+
+
+def _resolve_primitive(
+    primitive_id: PythonImportPath,
+    primitive_overrides: dict[PythonImportPath, Primitive] | None = None,
+) -> Result[Primitive, ErrorsList]:
+    if primitive_overrides is not None and primitive_id in primitive_overrides:
+        return Ok(primitive_overrides[primitive_id])
+
+    return resolve_primitive(primitive_id)
+
+
+def _ensure_markdown_constructible(
+    primitive: Primitive,
+    primitive_id: PythonImportPath | str | None = None,
+) -> Result[None, ErrorsList]:
+    if isinstance(primitive, MarkdownSectionMixin):
+        return Ok(None)
+
+    kind_label = f"'{primitive_id}'" if primitive_id is not None else repr(primitive)
+
+    return Err([world_errors.PrimitiveDoesNotSupportMarkdown(primitive_id=kind_label)])
+
+
+if TYPE_CHECKING:
+    from donna.world.config import SourceConfig as SourceConfigModel