agentic-development-framework 0.1.0__py3-none-any.whl

agentic_framework/requirements_manager/comparison.py

@@ -0,0 +1,129 @@
+"""Comparison and human-readable impact reporting for requirements versions."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass(frozen=True, slots=True)
+class ChangeSet:
+    """Added, modified, removed, and unchanged identifiers."""
+
+    added: tuple[str, ...]
+    modified: tuple[str, ...]
+    removed: tuple[str, ...]
+    unchanged: tuple[str, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class VersionComparison:
+    """File-level and requirement-level changes between two versions."""
+
+    previous_version: str
+    current_version: str
+    files: ChangeSet
+    requirements: ChangeSet
+    conflicts: tuple[str, ...]
+
+
+def _changes(previous: dict[str, str], current: dict[str, str]) -> ChangeSet:
+    previous_keys = set(previous)
+    current_keys = set(current)
+    shared = previous_keys & current_keys
+    return ChangeSet(
+        added=tuple(sorted(current_keys - previous_keys)),
+        modified=tuple(sorted(key for key in shared if previous[key] != current[key])),
+        removed=tuple(sorted(previous_keys - current_keys)),
+        unchanged=tuple(sorted(key for key in shared if previous[key] == current[key])),
+    )
+
+
+def compare_versions(
+    previous_manifest: dict[str, Any],
+    current_manifest: dict[str, Any],
+    previous_requirements: dict[str, Any],
+    current_requirements: dict[str, Any],
+) -> VersionComparison:
+    """Compare source hashes and extracted requirement hashes."""
+
+    previous_files = {item["path"]: item["sha256"] for item in previous_manifest["files"]}
+    current_files = {item["path"]: item["sha256"] for item in current_manifest["files"]}
+    previous_ids = {
+        item["id"]: item["text_sha256"] for item in previous_requirements["requirements"]
+    }
+    current_ids = {
+        item["id"]: item["text_sha256"] for item in current_requirements["requirements"]
+    }
+    return VersionComparison(
+        previous_version=previous_manifest["version"],
+        current_version=current_manifest["version"],
+        files=_changes(previous_files, current_files),
+        requirements=_changes(previous_ids, current_ids),
+        conflicts=tuple(current_manifest.get("conflicts", [])),
+    )
+
+
+def _section(title: str, values: tuple[str, ...]) -> list[str]:
+    lines = [f"### {title}", ""]
+    lines.extend(f"- `{value}`" for value in values)
+    if not values:
+        lines.append("- None")
+    lines.append("")
+    return lines
+
+
+def render_impact_report(comparison: VersionComparison) -> str:
+    """Render an actionable Markdown impact report."""
+
+    changed_ids = tuple(
+        sorted(
+            set(comparison.requirements.added)
+            | set(comparison.requirements.modified)
+            | set(comparison.requirements.removed)
+        )
+    )
+    lines = [
+        f"# Requirements impact: {comparison.previous_version} to {comparison.current_version}",
+        "",
+        "## Summary",
+        "",
+        "| Scope | Added | Modified | Removed | Unchanged |",
+        "|---|---:|---:|---:|---:|",
+        (
+            f"| Source files | {len(comparison.files.added)} | {len(comparison.files.modified)} | "
+            f"{len(comparison.files.removed)} | {len(comparison.files.unchanged)} |"
+        ),
+        (
+            f"| Requirement IDs | {len(comparison.requirements.added)} | "
+            f"{len(comparison.requirements.modified)} | {len(comparison.requirements.removed)} | "
+            f"{len(comparison.requirements.unchanged)} |"
+        ),
+        "",
+        "## Source file changes",
+        "",
+    ]
+    lines.extend(_section("Added", comparison.files.added))
+    lines.extend(_section("Modified", comparison.files.modified))
+    lines.extend(_section("Removed", comparison.files.removed))
+    lines.extend(["## Requirement changes", ""])
+    lines.extend(_section("Added", comparison.requirements.added))
+    lines.extend(_section("Modified", comparison.requirements.modified))
+    lines.extend(_section("Removed", comparison.requirements.removed))
+    lines.extend(_section("Conflicts requiring resolution", comparison.conflicts))
+    lines.extend(
+        [
+            "## Impact review",
+            "",
+            f"Requirement IDs requiring review: {', '.join(changed_ids) or 'none identified'}.",
+            "",
+            "- Documentation: review the SRS, feature descriptions, user flows, and traceability.",
+            "- Architecture: review components, data models, APIs, and decisions linked to "
+            "changed IDs.",
+            "- Implementation: review planned and active tasks linked to changed IDs.",
+            "- Tests: add, update, or retire tests linked to changed IDs.",
+            "- Approval: do not change approved decisions until this impact is reviewed.",
+            "",
+        ]
+    )
+    return "\n".join(lines)

agentic_framework/requirements_manager/extraction.py

@@ -0,0 +1,43 @@
+"""Small, explicit text-extraction boundary for requirement sources."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import StrEnum
+from pathlib import Path
+
+_TEXT_EXTENSIONS = {".csv", ".json", ".md", ".rst", ".tsv", ".txt", ".yaml", ".yml"}
+
+
+class ExtractionStatus(StrEnum):
+    """Result of trying to extract searchable text from a source file."""
+
+    EXTRACTED = "extracted"
+    UNSUPPORTED = "unsupported"
+    FAILED = "failed"
+
+
+@dataclass(frozen=True, slots=True)
+class ExtractionResult:
+    """Text and diagnostics produced by an extractor."""
+
+    status: ExtractionStatus
+    text: str | None = None
+    message: str | None = None
+
+
+def extract_text(path: Path) -> ExtractionResult:
+    """Extract UTF-8 text or report why extraction was not possible."""
+
+    extension = path.suffix.lower()
+    if extension not in _TEXT_EXTENSIONS:
+        label = extension or "files without an extension"
+        return ExtractionResult(
+            status=ExtractionStatus.UNSUPPORTED,
+            message=f"text extraction is not configured for {label}",
+        )
+    try:
+        text = path.read_text(encoding="utf-8-sig")
+    except (OSError, UnicodeError) as exc:
+        return ExtractionResult(status=ExtractionStatus.FAILED, message=str(exc))
+    return ExtractionResult(status=ExtractionStatus.EXTRACTED, text=text)

agentic_framework/requirements_manager/ingestion.py

@@ -0,0 +1,367 @@
+"""Atomic requirements ingestion and immutable version publication."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import mimetypes
+import os
+import re
+import shutil
+import tempfile
+from collections.abc import Iterator
+from contextlib import contextmanager
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from agentic_framework.exceptions import RequirementsError
+from agentic_framework.requirements_manager.comparison import (
+    VersionComparison,
+    compare_versions,
+    render_impact_report,
+)
+from agentic_framework.requirements_manager.extraction import ExtractionResult, extract_text
+from agentic_framework.requirements_manager.traceability import (
+    RequirementRecord,
+    extract_requirement_records,
+    traceability_document,
+)
+
+_VERSION_NAME = re.compile(r"version-(\d+)$")
+
+
+@dataclass(frozen=True, slots=True)
+class InventoryItem:
+    """One inbox source and its stable metadata."""
+
+    path: Path
+    relative_path: Path
+    sha256: str
+    size: int
+    media_type: str
+    extraction: ExtractionResult
+
+
+@dataclass(frozen=True, slots=True)
+class IngestionResult:
+    """Outcome of an ingestion attempt."""
+
+    version: str
+    version_path: Path
+    file_count: int
+    requirement_count: int
+    duplicate: bool
+    previous_version: str | None = None
+    impact_report: Path | None = None
+    comparison: VersionComparison | None = None
+
+
+def _sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    try:
+        with path.open("rb") as source:
+            for chunk in iter(lambda: source.read(1024 * 1024), b""):
+                digest.update(chunk)
+    except OSError as exc:
+        raise RequirementsError(f"cannot read requirement source {path}: {exc}") from exc
+    return digest.hexdigest()
+
+
+def _read_json(path: Path) -> dict[str, Any]:
+    try:
+        value = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, UnicodeError, json.JSONDecodeError) as exc:
+        raise RequirementsError(f"cannot read version metadata {path}: {exc}") from exc
+    if not isinstance(value, dict):
+        raise RequirementsError(f"version metadata must be a JSON object: {path}")
+    return value
+
+
+def _write_json(path: Path, value: dict[str, Any]) -> None:
+    path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+
+
+@contextmanager
+def _ingestion_lock(requirements_root: Path) -> Iterator[None]:
+    lock = requirements_root / ".ingestion.lock"
+    try:
+        descriptor = os.open(lock, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
+    except FileExistsError as exc:
+        raise RequirementsError(
+            "requirements ingestion is already running; remove stale lock only after "
+            f"verification: {lock}"
+        ) from exc
+    try:
+        os.write(descriptor, f"pid={os.getpid()}\n".encode())
+        os.close(descriptor)
+        yield
+    finally:
+        try:
+            os.close(descriptor)
+        except OSError:
+            pass
+        lock.unlink(missing_ok=True)
+
+
+class RequirementsManager:
+    """Version requirement sources without changing the inbox or prior versions."""
+
+    def __init__(self, project_root: Path) -> None:
+        self.project_root = project_root.expanduser().resolve()
+        self.requirements_root = self.project_root / "requirements"
+        self.inbox = self.requirements_root / "inbox"
+        self.versions = self.requirements_root / "versions"
+        self.reports = self.requirements_root / "reports"
+
+    def _prepare_directories(self) -> None:
+        self.inbox.mkdir(parents=True, exist_ok=True)
+        self.versions.mkdir(parents=True, exist_ok=True)
+        self.reports.mkdir(parents=True, exist_ok=True)
+
+    def inventory(self) -> tuple[InventoryItem, ...]:
+        """Hash and inspect every regular inbox file without modifying it."""
+
+        items: list[InventoryItem] = []
+        for path in sorted(self.inbox.rglob("*")):
+            if path.is_symlink():
+                raise RequirementsError(
+                    f"symbolic links are not accepted in requirements inbox: {path}"
+                )
+            if not path.is_file():
+                continue
+            relative = path.relative_to(self.inbox)
+            media_type = mimetypes.guess_type(path.name)[0] or "application/octet-stream"
+            items.append(
+                InventoryItem(
+                    path=path,
+                    relative_path=relative,
+                    sha256=_sha256(path),
+                    size=path.stat().st_size,
+                    media_type=media_type,
+                    extraction=extract_text(path),
+                )
+            )
+        if not items:
+            raise RequirementsError(f"no requirement files found in {self.inbox}")
+        return tuple(items)
+
+    def _published_versions(self) -> list[tuple[Path, dict[str, Any]]]:
+        published: list[tuple[Path, dict[str, Any]]] = []
+        directories = [
+            path
+            for path in self.versions.glob("version-*")
+            if path.is_dir() and _VERSION_NAME.fullmatch(path.name)
+        ]
+        directories.sort(key=lambda path: int(path.name.removeprefix("version-")))
+        for directory in directories:
+            manifest = directory / "manifest.json"
+            if manifest.is_file():
+                metadata = _read_json(manifest)
+                self._validate_published_version(directory, metadata)
+                published.append((directory, metadata))
+            elif any(directory.iterdir()):
+                raise RequirementsError(
+                    f"incomplete requirements version must be repaired: {directory}"
+                )
+        return published
+
+    def latest_version(self) -> dict[str, Any] | None:
+        """Return validated metadata for the latest published version."""
+
+        self._prepare_directories()
+        published = self._published_versions()
+        return published[-1][1] if published else None
+
+    @staticmethod
+    def _validate_published_version(directory: Path, manifest: dict[str, Any]) -> None:
+        if manifest.get("version") != directory.name or manifest.get("immutable") is not True:
+            raise RequirementsError(f"invalid immutable version manifest: {directory}")
+        files = manifest.get("files")
+        if not isinstance(files, list):
+            raise RequirementsError(f"invalid file inventory in version manifest: {directory}")
+        source_root = (directory / "source-files").resolve()
+        for item in files:
+            if not isinstance(item, dict) or not isinstance(item.get("stored_path"), str):
+                raise RequirementsError(f"invalid source record in version manifest: {directory}")
+            stored = (directory / item["stored_path"]).resolve()
+            if not stored.is_relative_to(source_root) or not stored.is_file():
+                raise RequirementsError(f"missing or unsafe immutable source copy: {stored}")
+            if _sha256(stored) != item.get("sha256"):
+                raise RequirementsError(f"immutable source copy was modified: {stored}")
+        artifacts = manifest.get("artifacts")
+        if not isinstance(artifacts, dict):
+            raise RequirementsError(f"invalid artifact inventory in version manifest: {directory}")
+        for relative, expected_hash in artifacts.items():
+            artifact = (directory / relative).resolve()
+            if not artifact.is_relative_to(directory.resolve()) or not artifact.is_file():
+                raise RequirementsError(f"missing or unsafe immutable version artifact: {artifact}")
+            if _sha256(artifact) != expected_hash:
+                raise RequirementsError(f"immutable version artifact was modified: {artifact}")
+
+    @staticmethod
+    def _signature(items: tuple[InventoryItem, ...]) -> dict[str, str]:
+        return {item.relative_path.as_posix(): item.sha256 for item in items}
+
+    @staticmethod
+    def _manifest_signature(manifest: dict[str, Any]) -> dict[str, str]:
+        return {item["path"]: item["sha256"] for item in manifest["files"]}
+
+    def _next_target(self, published: list[tuple[Path, dict[str, Any]]]) -> tuple[str, Path]:
+        number = (
+            int(published[-1][0].name.removeprefix("version-")) + 1 if published else 1
+        )
+        version = f"version-{number:03d}"
+        target = self.versions / version
+        if target.exists() and any(target.iterdir()):
+            raise RequirementsError(f"requirements version target already contains data: {target}")
+        return version, target
+
+    @staticmethod
+    def _requirements_markdown(items: tuple[InventoryItem, ...]) -> str:
+        lines = ["# Consolidated requirements", ""]
+        for item in items:
+            relative = item.relative_path.as_posix()
+            lines.extend([f"## Source: `{relative}`", ""])
+            if item.extraction.text is not None:
+                lines.extend([item.extraction.text.rstrip(), ""])
+            else:
+                lines.extend(
+                    [
+                        f"Extraction status: **{item.extraction.status.value}**.",
+                        "",
+                        item.extraction.message or "No diagnostic was provided.",
+                        "",
+                    ]
+                )
+        return "\n".join(lines).rstrip() + "\n"
+
+    @staticmethod
+    def _requirements_index(records: dict[str, RequirementRecord]) -> dict[str, object]:
+        return {
+            "schema_version": 1,
+            "requirements": [
+                {
+                    "id": record.requirement_id,
+                    "source": record.source,
+                    "text": record.text,
+                    "text_sha256": record.sha256,
+                }
+                for record in sorted(records.values(), key=lambda item: item.requirement_id)
+            ],
+        }
+
+    def _publish(
+        self,
+        items: tuple[InventoryItem, ...],
+        version: str,
+        target: Path,
+        previous_version: str | None,
+    ) -> tuple[dict[str, Any], dict[str, object]]:
+        extracted = {
+            item.relative_path.as_posix(): item.extraction.text
+            for item in items
+            if item.extraction.text is not None
+        }
+        records, conflicts = extract_requirement_records(extracted)
+        manifest: dict[str, Any] = {
+            "schema_version": 1,
+            "version": version,
+            "created_at": datetime.now(UTC).isoformat(),
+            "previous_version": previous_version,
+            "immutable": True,
+            "files": [
+                {
+                    "path": item.relative_path.as_posix(),
+                    "stored_path": f"source-files/{item.relative_path.as_posix()}",
+                    "sha256": item.sha256,
+                    "size": item.size,
+                    "media_type": item.media_type,
+                    "extraction": {
+                        "status": item.extraction.status.value,
+                        "message": item.extraction.message,
+                    },
+                }
+                for item in items
+            ],
+            "conflicts": list(conflicts),
+        }
+        requirements_index = self._requirements_index(records)
+        with tempfile.TemporaryDirectory(prefix=".adf-version-", dir=self.versions) as temporary:
+            staging = Path(temporary)
+            for item in items:
+                destination = staging / "source-files" / item.relative_path
+                destination.parent.mkdir(parents=True, exist_ok=True)
+                shutil.copyfile(item.path, destination)
+                if _sha256(destination) != item.sha256:
+                    raise RequirementsError(f"copied source failed hash verification: {item.path}")
+            _write_json(staging / "requirements-index.json", requirements_index)
+            _write_json(staging / "traceability.json", traceability_document(records))
+            (staging / "requirements.md").write_text(
+                self._requirements_markdown(items), encoding="utf-8"
+            )
+            manifest["artifacts"] = {
+                name: _sha256(staging / name)
+                for name in (
+                    "requirements-index.json",
+                    "requirements.md",
+                    "traceability.json",
+                )
+            }
+            _write_json(staging / "manifest.json", manifest)
+            if target.exists():
+                target.rmdir()
+            os.replace(staging, target)
+        return manifest, requirements_index
+
+    def _write_impact_report(self, comparison: VersionComparison) -> Path:
+        report = self.reports / (
+            f"{comparison.previous_version}-to-{comparison.current_version}-impact.md"
+        )
+        content = render_impact_report(comparison)
+        if report.exists():
+            if report.read_text(encoding="utf-8") != content:
+                raise RequirementsError(f"refusing to overwrite existing impact report: {report}")
+            return report
+        report.write_text(content, encoding="utf-8")
+        return report
+
+    def ingest(self) -> IngestionResult:
+        """Publish the inbox as the next immutable version or identify a duplicate."""
+
+        self._prepare_directories()
+        with _ingestion_lock(self.requirements_root):
+            items = self.inventory()
+            published = self._published_versions()
+            latest = published[-1] if published else None
+            if latest and self._manifest_signature(latest[1]) == self._signature(items):
+                index = _read_json(latest[0] / "requirements-index.json")
+                return IngestionResult(
+                    version=latest[1]["version"],
+                    version_path=latest[0],
+                    file_count=len(items),
+                    requirement_count=len(index["requirements"]),
+                    duplicate=True,
+                    previous_version=latest[1].get("previous_version"),
+                )
+
+            version, target = self._next_target(published)
+            previous_version = latest[1]["version"] if latest else None
+            manifest, index = self._publish(items, version, target, previous_version)
+            comparison = None
+            impact_report = None
+            if latest:
+                previous_index = _read_json(latest[0] / "requirements-index.json")
+                comparison = compare_versions(latest[1], manifest, previous_index, index)
+                impact_report = self._write_impact_report(comparison)
+            return IngestionResult(
+                version=version,
+                version_path=target,
+                file_count=len(items),
+                requirement_count=len(index["requirements"]),
+                duplicate=False,
+                previous_version=previous_version,
+                impact_report=impact_report,
+                comparison=comparison,
+            )

agentic_framework/requirements_manager/traceability.py

@@ -0,0 +1,65 @@
+"""Requirement-ID extraction and initial traceability records."""
+
+from __future__ import annotations
+
+import hashlib
+import re
+from dataclasses import dataclass
+
+_REQUIREMENT_HEADING = re.compile(r"(?m)^\s*(REQ-\d+)\b[^\n]*")
+
+
+@dataclass(frozen=True, slots=True)
+class RequirementRecord:
+    """One identified requirement and its source location."""
+
+    requirement_id: str
+    source: str
+    text: str
+
+    @property
+    def sha256(self) -> str:
+        return hashlib.sha256(self.text.encode("utf-8")).hexdigest()
+
+
+def extract_requirement_records(
+    extracted_sources: dict[str, str],
+) -> tuple[dict[str, RequirementRecord], tuple[str, ...]]:
+    """Extract requirement blocks and report duplicate IDs with conflicting text."""
+
+    records: dict[str, RequirementRecord] = {}
+    conflicts: set[str] = set()
+    for source, text in sorted(extracted_sources.items()):
+        matches = list(_REQUIREMENT_HEADING.finditer(text))
+        for index, match in enumerate(matches):
+            end = matches[index + 1].start() if index + 1 < len(matches) else len(text)
+            block = text[match.start() : end].strip().splitlines()
+            normalized = "\n".join(line.rstrip() for line in block)
+            requirement_id = match.group(1).upper()
+            record = RequirementRecord(requirement_id, source, normalized)
+            existing = records.get(requirement_id)
+            if existing is None:
+                records[requirement_id] = record
+            elif existing.text != record.text:
+                conflicts.add(requirement_id)
+    return records, tuple(sorted(conflicts))
+
+
+def traceability_document(records: dict[str, RequirementRecord]) -> dict[str, object]:
+    """Create the initial traceability shape extended by later workflow phases."""
+
+    return {
+        "schema_version": 1,
+        "requirements": [
+            {
+                "id": record.requirement_id,
+                "source": record.source,
+                "text_sha256": record.sha256,
+                "srs_sections": [],
+                "architecture_components": [],
+                "tasks": [],
+                "tests": [],
+            }
+            for record in sorted(records.values(), key=lambda item: item.requirement_id)
+        ],
+    }

agentic_framework/state/__init__.py

@@ -0,0 +1,5 @@
+"""SQLite-backed runtime state and recovery."""
+
+from agentic_framework.state.database import RuntimeStore
+
+__all__ = ["RuntimeStore"]