qwen-sci 0.2.2__py3-none-any.whl

src/agents/research_plan_author/source_registry.py

@@ -0,0 +1,208 @@
+"""Build a global, source-bounded knowledge base for Research Plan Authoring."""
+
+from __future__ import annotations
+
+from copy import deepcopy
+import hashlib
+import json
+from typing import Any, Mapping
+
+
+def _mapping(value: object) -> dict[str, Any]:
+    return dict(value) if isinstance(value, Mapping) else {}
+
+
+def _text(value: object) -> str:
+    return str(value or "").strip()
+
+
+def _append_unique(records: list[dict[str, Any]], record: dict[str, Any], *, key: str) -> None:
+    value = _text(record.get(key))
+    if value and not any(_text(existing.get(key)) == value for existing in records):
+        records.append(record)
+
+
+def _citation_key(source_id: str) -> str:
+    """Keep BibTeX keys stable and syntax-safe without exposing a raw URL/DOI."""
+
+    digest = hashlib.sha256(source_id.encode("utf-8")).hexdigest()[:16]
+    return f"cite_{digest}"
+
+
+def _source_items(items: object, *, prefix: str) -> list[dict[str, Any]]:
+    values = items if isinstance(items, list) else []
+    records: list[dict[str, Any]] = []
+    for index, item in enumerate(values, start=1):
+        canonical = json.dumps(item, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False)
+        records.append(
+            {
+                "source_item_id": f"{prefix}-{index}",
+                "source_item_sha256": hashlib.sha256(canonical.encode("utf-8")).hexdigest(),
+                "original_item": deepcopy(item),
+            }
+        )
+    return records
+
+
+def build_frozen_source_registry(preparation: Mapping[str, Any]) -> dict[str, Any]:
+    """Expose every already-known source; an LLM cannot mint bibliography data."""
+
+    bundle = _mapping(preparation.get("source_bundle"))
+    handoff = _mapping(bundle.get("author_context"))
+    compact = _mapping(handoff.get("source_registry"))
+    evidence_cards_by_id = {
+        str(card_id): deepcopy(dict(card))
+        for card_id, card in (_mapping(compact.get("evidence_cards_by_id"))).items()
+        if isinstance(card, Mapping)
+    }
+    allowed_source_ids = [
+        _text(source_id)
+        for source_id in compact.get("allowed_source_ids") or []
+        if _text(source_id)
+    ]
+    citations = [
+        deepcopy(dict(citation))
+        for citation in compact.get("citation_registry") or []
+        if isinstance(citation, Mapping)
+    ]
+    survey_sources = _mapping(bundle.get("survey_sources"))
+    survey_artifacts = _mapping(survey_sources.get("artifacts"))
+    survey_anchor_ids = [f"survey:{name}" for name in sorted(survey_artifacts)]
+    survey_markdown = _mapping(survey_artifacts.get("survey_markdown"))
+    for excerpt in survey_markdown.get("excerpts") or []:
+        if not isinstance(excerpt, Mapping):
+            continue
+        anchor_id = _text(excerpt.get("anchor_id"))
+        if anchor_id.startswith("survey:survey_markdown#") and anchor_id not in survey_anchor_ids:
+            survey_anchor_ids.append(anchor_id)
+    unknown_items = _source_items(handoff.get("unknown_items"), prefix="unknown")
+    review_items = _source_items(handoff.get("review_items"), prefix="review")
+    return {
+        "schema_version": "research_plan_author_source_registry_v3",
+        "allowed_source_ids": allowed_source_ids,
+        "allowed_survey_anchor_ids": survey_anchor_ids,
+        "evidence_cards_by_id": evidence_cards_by_id,
+        "citation_registry": deepcopy(citations),
+        "unknown_items": unknown_items,
+        "review_items": review_items,
+    }
+
+
+_RECOMMENDED_SLOTS_BY_SECTION = {
+    "introduction": {"research_object_measurability", "mechanism", "boundary_conditions"},
+    "survey_and_research_gap": {"research_object_measurability", "mechanism", "boundary_conditions"},
+    "study_design_and_methods": {
+        "study_design",
+        "measurement_calibration",
+        "comparison_controls",
+        "statistics_bias",
+    },
+    "computational_evaluation_protocol": {
+        "study_design",
+        "measurement_calibration",
+        "comparison_controls",
+        "statistics_bias",
+    },
+    "expected_outcomes": {"mechanism", "comparison_controls", "statistics_bias"},
+    "risk_limitations_and_review": {"risk_ethics_reproducibility", "boundary_conditions"},
+    "appendix_evidence_and_review": None,
+}
+
+
+def build_authoring_knowledge_base(
+    preparation: Mapping[str, Any],
+    source_registry: Mapping[str, Any],
+) -> dict[str, Any]:
+    """Assemble every frozen upstream artifact into one writer-visible knowledge base.
+
+    The knowledge base is deliberately broader than a route recommendation.  It
+    preserves provenance by exposing canonical IDs and metadata, but it never
+    turns an evidence slot into a permission boundary for scholarly synthesis.
+    """
+
+    bundle = _mapping(preparation.get("source_bundle"))
+    author_context = _mapping(bundle.get("author_context"))
+    return {
+        "schema_version": "research_plan_authoring_knowledge_base_v1",
+        "theory_spine": deepcopy(_mapping(preparation.get("theory_spine"))),
+        "source_catalog": {
+            "allowed_source_ids": list(source_registry.get("allowed_source_ids") or []),
+            "citation_registry": deepcopy(list(source_registry.get("citation_registry") or [])),
+            "evidence_cards_by_id": deepcopy(_mapping(source_registry.get("evidence_cards_by_id"))),
+            "survey_anchor_ids": list(source_registry.get("allowed_survey_anchor_ids") or []),
+        },
+        "upstream_artifacts": {
+            "selected_direction": deepcopy(author_context.get("selected_direction")),
+            "research_design": deepcopy(author_context.get("research_design")),
+            "hypothesis_mapping": deepcopy(author_context.get("hypothesis_mapping")),
+            "formal_reasoning": deepcopy(author_context.get("formal_reasoning")),
+            "counterexample_analysis": deepcopy(author_context.get("counterexample_analysis")),
+            "outcome_branches": deepcopy(author_context.get("outcome_branches")),
+            "reasoning_context": deepcopy(author_context.get("reasoning_context")),
+            "variables_and_operationalization": deepcopy(author_context.get("variables_and_operationalization")),
+            "idea_evolution": deepcopy(_mapping(bundle.get("idea_evolution"))),
+            "survey_binding": deepcopy(_mapping(bundle.get("survey_binding"))),
+        },
+        "unknown_items": deepcopy(list(source_registry.get("unknown_items") or [])),
+        "review_items": deepcopy(list(source_registry.get("review_items") or [])),
+    }
+
+
+def source_registry_for_route(source_registry: Mapping[str, Any], route: Mapping[str, Any]) -> dict[str, Any]:
+    """Return the global catalog plus non-binding relevance recommendations.
+
+    Earlier versions used ``claim_slot`` to hide sources from most sections.
+    That made source traceability a writing prohibition and caused shallow,
+    repetitive prose.  The writer may now use any canonical source; slots only
+    help it start from a sensible evidence cluster.
+    """
+
+    section_id = _text(route.get("section_id")).casefold()
+    recommended_slots = _RECOMMENDED_SLOTS_BY_SECTION.get(section_id, set())
+    cards = _mapping(source_registry.get("evidence_cards_by_id"))
+    if recommended_slots is None:
+        recommended_source_ids = list(source_registry.get("allowed_source_ids") or [])
+    else:
+        recommended_source_ids = sorted(
+            {
+                _text(_mapping(card).get("source_id"))
+                for card in cards.values()
+                if _text(_mapping(card).get("source_id"))
+                and _text(_mapping(card).get("claim_slot")).casefold() in recommended_slots
+            }
+        )
+    return {
+        "schema_version": _text(source_registry.get("schema_version")),
+        "allowed_source_ids": list(source_registry.get("allowed_source_ids") or []),
+        "allowed_survey_anchor_ids": list(source_registry.get("allowed_survey_anchor_ids") or []),
+        "evidence_cards_by_id": deepcopy(cards),
+        "citation_registry": deepcopy(list(source_registry.get("citation_registry") or [])),
+        "unknown_items": deepcopy(list(source_registry.get("unknown_items") or [])),
+        "review_items": deepcopy(list(source_registry.get("review_items") or [])),
+        "recommended_source_ids": recommended_source_ids,
+        "recommended_claim_slots": sorted(recommended_slots or set()),
+        "authoring_knowledge_base": deepcopy(_mapping(source_registry.get("authoring_knowledge_base"))),
+    }
+
+
+def source_registry_for_blueprint_section(
+    source_registry: Mapping[str, Any],
+    route: Mapping[str, Any],
+    blueprint_section: Mapping[str, Any],
+) -> dict[str, Any]:
+    """Preserve global source access after Blueprint planning.
+
+    ``allowed_source_ids`` in historic Blueprint artifacts is retained only for
+    cache compatibility and observability.  It must never narrow the sources
+    available to the section writer.
+    """
+
+    return source_registry_for_route(source_registry, route)
+
+
+__all__ = [
+    "build_frozen_source_registry",
+    "build_authoring_knowledge_base",
+    "source_registry_for_blueprint_section",
+    "source_registry_for_route",
+]

src/agents/research_plan_author/survey_source_loader.py

@@ -0,0 +1,137 @@
+"""Load the Survey artifacts required by a complete Research Plan Author run."""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+from typing import Any, Mapping
+
+from src.pipeline.survey_idea_loader import SurveyIdeaLoadError, load_survey_idea_context
+
+
+class SurveyAuthorSourceError(ValueError):
+    """Raised when verified Survey sources cannot be used for Author writing."""
+
+
+_REQUIRED_ARTIFACTS = (
+    "idea_handoff",
+    "gap_ledger",
+    "project_context",
+    "survey_json",
+    "survey_markdown",
+    "claim_traceability",
+    "evidence_plan",
+)
+_MARKDOWN_HEADING = re.compile(r"^#{1,6}\s+.+?\s*$", re.MULTILINE)
+
+
+def _mapping(value: object) -> dict[str, Any]:
+    return dict(value) if isinstance(value, Mapping) else {}
+
+
+def _text(value: object) -> str:
+    return str(value or "").strip()
+
+
+def _safe_artifact_path(base_dir: Path, entry: Mapping[str, Any], name: str) -> Path:
+    relative_path = _text(entry.get("path"))
+    if not relative_path:
+        raise SurveyAuthorSourceError(f"Survey manifest is missing artifact entry: {name}")
+    candidate = (base_dir / relative_path).resolve()
+    try:
+        candidate.relative_to(base_dir)
+    except ValueError as error:
+        raise SurveyAuthorSourceError(f"Survey artifact path escapes manifest directory: {name}") from error
+    if not candidate.is_file():
+        raise SurveyAuthorSourceError(f"Survey artifact is missing: {candidate}")
+    return candidate
+
+
+def _read_json(path: Path, name: str) -> dict[str, Any]:
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except OSError as error:
+        raise SurveyAuthorSourceError(f"Cannot read Survey artifact '{name}': {error}") from error
+    except json.JSONDecodeError as error:
+        raise SurveyAuthorSourceError(f"Survey artifact '{name}' is not valid JSON: {error}") from error
+    if not isinstance(payload, Mapping):
+        raise SurveyAuthorSourceError(f"Survey artifact '{name}' must contain one JSON object")
+    return dict(payload)
+
+
+def _survey_markdown_excerpts(markdown: str) -> list[dict[str, Any]]:
+    """Split verified Survey Markdown into complete, source-addressable excerpts."""
+
+    starts = [match.start() for match in _MARKDOWN_HEADING.finditer(markdown)]
+    boundaries = [0, *starts] if starts and starts[0] else starts
+    if not boundaries:
+        boundaries = [0] if markdown.strip() else []
+    excerpts: list[dict[str, Any]] = []
+    for ordinal, start in enumerate(boundaries, start=1):
+        end = boundaries[ordinal] if ordinal < len(boundaries) else len(markdown)
+        text = markdown[start:end].strip()
+        if not text:
+            continue
+        first_line = text.splitlines()[0].strip()
+        heading = first_line.lstrip("#").strip() if first_line.startswith("#") else "Survey preamble"
+        excerpts.append(
+            {
+                "anchor_id": f"survey:survey_markdown#section-{ordinal:03d}",
+                "heading": heading,
+                "ordinal": ordinal,
+                "text": text,
+            }
+        )
+    return excerpts
+
+
+def load_verified_survey_sources(source: str | Path) -> dict[str, Any]:
+    """Return a verified, source-addressable Survey bundle for Author composition."""
+
+    try:
+        context = load_survey_idea_context(source)
+    except SurveyIdeaLoadError as error:
+        raise SurveyAuthorSourceError(str(error)) from error
+    if context.legacy:
+        raise SurveyAuthorSourceError(
+            "Research Plan Author requires a completed, verified survey_manifest.json; legacy Survey directories are unsupported"
+        )
+    artifact_entries = _mapping(context.manifest.get("artifacts"))
+    artifact_paths: dict[str, str] = {}
+    artifacts: dict[str, Any] = {}
+    for name in _REQUIRED_ARTIFACTS:
+        path = _safe_artifact_path(context.base_dir, _mapping(artifact_entries.get(name)), name)
+        artifact_paths[name] = str(path)
+        if name == "survey_markdown":
+            artifacts[name] = {
+                "path": str(path),
+                "content_available": True,
+                "excerpts": _survey_markdown_excerpts(context.survey_markdown),
+            }
+        else:
+            artifacts[name] = _read_json(path, name)
+    for name, raw_entry in artifact_entries.items():
+        if name in artifacts:
+            continue
+        entry = _mapping(raw_entry)
+        relative_path = _text(entry.get("path"))
+        if not relative_path:
+            continue
+        optional_path = _safe_artifact_path(context.base_dir, entry, str(name))
+        artifact_paths[str(name)] = str(optional_path)
+    return {
+        "schema_version": "research_plan_author_survey_sources_v1",
+        "manifest_path": str(context.manifest_path),
+        "base_dir": str(context.base_dir),
+        "survey_run_id": context.survey_run_id,
+        "project_id": context.project_id,
+        "project_context_fingerprint": context.project_context_fingerprint,
+        "topic": context.topic,
+        "manifest": dict(context.manifest),
+        "artifacts": artifacts,
+        "artifact_paths": artifact_paths,
+    }
+
+
+__all__ = ["SurveyAuthorSourceError", "load_verified_survey_sources"]

src/agents/research_plan_author/template_adapter.py

@@ -0,0 +1,173 @@
+"""Copy and adapt declared LaTeX templates without changing their source tree."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+import os
+from pathlib import Path
+import shutil
+import tempfile
+from typing import Mapping
+
+from .template_profile import TemplateInsertion, TemplateProfile
+
+
+class TemplateAdapterError(RuntimeError):
+    """Raised when an explicit template cannot be copied or adapted safely."""
+
+
+@dataclass(frozen=True)
+class MaterializedTemplate:
+    profile_id: str
+    source_dir: Path
+    project_dir: Path
+    main_tex: Path
+    generated_bib: Path
+
+
+def _resolve_relative(root: Path, value: str, *, label: str) -> Path:
+    relative = Path(value)
+    if relative.is_absolute() or ".." in relative.parts:
+        raise TemplateAdapterError(f"{label} must be a relative path inside the template")
+    candidate = (root / relative).resolve()
+    try:
+        candidate.relative_to(root)
+    except ValueError as error:
+        raise TemplateAdapterError(f"{label} escapes the template root") from error
+    return candidate
+
+
+def _reject_symlinks(root: Path) -> None:
+    for candidate in root.rglob("*"):
+        if candidate.is_symlink():
+            raise TemplateAdapterError(
+                f"template contains unsupported symlink '{candidate.relative_to(root)}'; copy real supporting files instead"
+            )
+
+
+def _find_exact(content: str, needle: str, *, label: str) -> int:
+    count = content.count(needle)
+    if count != 1:
+        raise TemplateAdapterError(
+            f"template insertion '{label}' must occur exactly once; found {count} occurrences"
+        )
+    return content.index(needle)
+
+
+def _replacement_span(content: str, insertion: TemplateInsertion, *, label: str) -> tuple[int, int]:
+    if insertion.kind == "marker":
+        start = _find_exact(content, insertion.marker, label=label)
+        return start, start + len(insertion.marker)
+    if insertion.kind != "region":
+        raise TemplateAdapterError(f"template insertion '{label}' has unsupported kind '{insertion.kind}'")
+    start_anchor = _find_exact(content, insertion.start, label=f"{label}.start")
+    end_anchor = _find_exact(content, insertion.end, label=f"{label}.end")
+    if end_anchor <= start_anchor:
+        raise TemplateAdapterError(f"template insertion '{label}' end anchor must follow its start anchor")
+    start = start_anchor if insertion.include_start else start_anchor + len(insertion.start)
+    end = end_anchor + len(insertion.end) if insertion.include_end else end_anchor
+    if end <= start:
+        raise TemplateAdapterError(f"template insertion '{label}' resolves to an empty or reversed span")
+    return start, end
+
+
+class TemplateAdapter:
+    """Materialize a user-owned template and apply one prevalidated replacement set."""
+
+    def materialize(
+        self,
+        template_dir: str | Path,
+        project_dir: str | Path,
+        profile: TemplateProfile,
+    ) -> MaterializedTemplate:
+        source_dir = Path(template_dir).expanduser().resolve()
+        destination = Path(project_dir).expanduser().resolve()
+        if not source_dir.is_dir():
+            raise TemplateAdapterError(f"template directory does not exist: {source_dir}")
+        _reject_symlinks(source_dir)
+        source_main = _resolve_relative(source_dir, profile.main_tex, label="template main TeX path")
+        if not source_main.is_file():
+            raise TemplateAdapterError(
+                f"template profile '{profile.profile_id}' main TeX file does not exist: {profile.main_tex}"
+            )
+        reserved_destination = False
+        try:
+            destination.parent.mkdir(parents=True, exist_ok=True)
+            destination.mkdir(exist_ok=False)
+            reserved_destination = True
+            shutil.copytree(source_dir, destination, dirs_exist_ok=True)
+        except OSError as error:
+            try:
+                if reserved_destination and destination.exists():
+                    shutil.rmtree(destination, ignore_errors=True)
+            except OSError:
+                pass
+            raise TemplateAdapterError(f"cannot copy template to output-owned project: {error}") from error
+        return MaterializedTemplate(
+            profile_id=profile.profile_id,
+            source_dir=source_dir,
+            project_dir=destination,
+            main_tex=_resolve_relative(destination, profile.main_tex, label="template main TeX path"),
+            generated_bib=_resolve_relative(destination, profile.generated_bib, label="generated BibTeX path"),
+        )
+
+    def apply(
+        self,
+        materialized: MaterializedTemplate,
+        profile: TemplateProfile,
+        replacements: Mapping[str, str],
+    ) -> Path:
+        required = set(profile.insertions)
+        missing = sorted(required - set(replacements))
+        unexpected = sorted(set(replacements) - required)
+        if missing or unexpected:
+            details: list[str] = []
+            if missing:
+                details.append("missing replacements: " + ", ".join(missing))
+            if unexpected:
+                details.append("unexpected replacements: " + ", ".join(unexpected))
+            raise TemplateAdapterError("; ".join(details))
+        try:
+            content = materialized.main_tex.read_text(encoding="utf-8")
+        except OSError as error:
+            raise TemplateAdapterError(f"cannot read copied template main TeX file: {error}") from error
+        spans: list[tuple[int, int, str, str]] = []
+        for label, insertion in profile.insertions.items():
+            start, end = _replacement_span(content, insertion, label=label)
+            spans.append((start, end, label, str(replacements[label])))
+        for index, insertion in enumerate(profile.discard_regions):
+            start, end = _replacement_span(content, insertion, label=f"discard_regions[{index}]")
+            spans.append((start, end, f"discard_regions[{index}]", ""))
+        ordered = sorted(spans)
+        for previous, current in zip(ordered, ordered[1:]):
+            if previous[1] > current[0]:
+                raise TemplateAdapterError(
+                    f"template profile '{profile.profile_id}' contains overlapping insertion regions "
+                    f"('{previous[2]}' and '{current[2]}')"
+                )
+        rendered = content
+        for start, end, _label, replacement in sorted(spans, reverse=True):
+            rendered = rendered[:start] + replacement + rendered[end:]
+        temporary: Path | None = None
+        try:
+            descriptor, raw_path = tempfile.mkstemp(
+                prefix=f".{materialized.main_tex.name}.",
+                suffix=".tmp",
+                dir=materialized.main_tex.parent,
+                text=True,
+            )
+            temporary = Path(raw_path)
+            with os.fdopen(descriptor, "w", encoding="utf-8", newline="\n") as handle:
+                handle.write(rendered)
+                handle.flush()
+                os.fsync(handle.fileno())
+            os.replace(temporary, materialized.main_tex)
+        except OSError as error:
+            raise TemplateAdapterError(f"cannot write adapted TeX file: {error}") from error
+        finally:
+            if temporary is not None:
+                temporary.unlink(missing_ok=True)
+        return materialized.main_tex
+
+
+__all__ = ["MaterializedTemplate", "TemplateAdapter", "TemplateAdapterError"]