briefloop 0.11.12__py3-none-any.whl

multi_agent_brief/contracts/schemas/source_evidence_pack_manifest.py

@@ -0,0 +1,233 @@
+"""Contract for durable source evidence pack manifests."""
+
+from __future__ import annotations
+
+import re
+from typing import Any, ClassVar
+
+from multi_agent_brief.contracts.base import Contract, SchemaRegistry
+from multi_agent_brief.contracts.errors import FieldViolation
+from multi_agent_brief.contracts.source_metadata import (
+    VALID_RETRIEVAL_SOURCE_TYPES,
+    VALID_SOURCE_CATEGORIES,
+    VALID_UNDERLYING_EVIDENCE_TYPES,
+    retrieval_source_type_error,
+    source_category_error,
+    underlying_evidence_type_error,
+)
+
+SOURCE_EVIDENCE_PACK_MANIFEST_SCHEMA_VERSION = "mabw.source_evidence_pack_manifest.v1"
+SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
+
+
+@SchemaRegistry.register
+class SourceEvidencePackManifestContract(Contract):
+    """Validate source evidence pack manifest shape.
+
+    This contract validates deterministic source-pack bookkeeping only. It does
+    not judge whether a source supports a claim.
+    """
+
+    schema_id: ClassVar[str] = "source_evidence_pack_manifest"
+    schema_version: ClassVar[str] = "v1"
+
+    @classmethod
+    def json_schema(cls) -> dict[str, Any]:
+        return {
+            "type": "object",
+            "required": ["schema_version", "records", "pack_sha256"],
+            "properties": {
+                "schema_version": {
+                    "type": "string",
+                    "enum": [SOURCE_EVIDENCE_PACK_MANIFEST_SCHEMA_VERSION],
+                },
+                "source": {"type": "string"},
+                "source_config_path": {"type": "string"},
+                "record_count": {"type": "integer", "minimum": 1},
+                "error_count": {"type": "integer", "minimum": 0},
+                "pack_sha256": {"type": "string", "pattern": SHA256_RE.pattern},
+                "durable_provider_names": {
+                    "type": "array",
+                    "items": {"type": "string"},
+                },
+                "records": {
+                    "type": "array",
+                    "minItems": 1,
+                    "items": {
+                        "type": "object",
+                        "required": ["source_id", "path", "sha256", "size_bytes"],
+                        "properties": {
+                            "source_id": {"type": "string"},
+                            "path": {"type": "string"},
+                            "sha256": {"type": "string", "pattern": SHA256_RE.pattern},
+                            "size_bytes": {"type": "integer", "minimum": 1},
+                            "source_url": {"type": "string"},
+                            "source_title": {"type": "string"},
+                            "publisher": {"type": "string"},
+                            "source_type": {"type": "string"},
+                            "source_category": {
+                                "type": "string",
+                                "enum": sorted(VALID_SOURCE_CATEGORIES),
+                            },
+                            "retrieval_source_type": {
+                                "type": "string",
+                                "enum": sorted(VALID_RETRIEVAL_SOURCE_TYPES),
+                            },
+                            "underlying_evidence_type": {
+                                "type": "string",
+                                "enum": sorted(VALID_UNDERLYING_EVIDENCE_TYPES),
+                            },
+                            "raw_underlying_evidence_type": {"type": "string"},
+                        },
+                        "additionalProperties": True,
+                    },
+                },
+                "provider_errors": {
+                    "type": "array",
+                    "items": {"type": "object"},
+                },
+                "non_goals": {
+                    "type": "array",
+                    "items": {"type": "string"},
+                },
+            },
+            "additionalProperties": True,
+        }
+
+    @classmethod
+    def validate(cls, data: dict[str, Any]) -> list[FieldViolation]:
+        if not isinstance(data, dict):
+            return [FieldViolation(field="<root>", error="must be an object")]
+
+        violations: list[FieldViolation] = []
+        schema_version = data.get("schema_version")
+        if not _non_empty_string(schema_version):
+            violations.append(FieldViolation(field="schema_version", error="required field is missing"))
+        elif schema_version != SOURCE_EVIDENCE_PACK_MANIFEST_SCHEMA_VERSION:
+            violations.append(
+                FieldViolation(
+                    field="schema_version",
+                    error=f"must be {SOURCE_EVIDENCE_PACK_MANIFEST_SCHEMA_VERSION}",
+                )
+            )
+
+        records = data.get("records")
+        if not isinstance(records, list) or not records:
+            violations.append(FieldViolation(field="records", error="must be a non-empty list"))
+        else:
+            seen_paths: set[str] = set()
+            seen_source_ids: set[str] = set()
+            for idx, record in enumerate(records):
+                violations.extend(
+                    _validate_record(
+                        record,
+                        idx=idx,
+                        seen_paths=seen_paths,
+                        seen_source_ids=seen_source_ids,
+                    )
+                )
+            record_count = data.get("record_count")
+            if record_count is not None and record_count != len(records):
+                violations.append(
+                    FieldViolation(
+                        field="record_count",
+                        error=f"must equal records length:{len(records)}",
+                    )
+                )
+
+        pack_sha = data.get("pack_sha256")
+        if not _valid_sha256(pack_sha):
+            violations.append(FieldViolation(field="pack_sha256", error="must be a lowercase sha256 hex digest"))
+
+        for field in ("record_count", "error_count"):
+            value = data.get(field)
+            if value is not None and (type(value) is not int or value < (1 if field == "record_count" else 0)):
+                violations.append(FieldViolation(field=field, error="must be a non-negative integer"))
+
+        provider_errors = data.get("provider_errors")
+        if provider_errors is not None and not isinstance(provider_errors, list):
+            violations.append(FieldViolation(field="provider_errors", error="must be a list"))
+        elif provider_errors is not None:
+            error_count = data.get("error_count")
+            if error_count is not None and error_count != len(provider_errors):
+                violations.append(
+                    FieldViolation(
+                        field="error_count",
+                        error=f"must equal provider_errors length:{len(provider_errors)}",
+                    )
+                )
+
+        return violations
+
+    @classmethod
+    def migrate(cls, data: dict[str, Any], from_version: str) -> dict[str, Any]:
+        return dict(data)
+
+
+def _validate_record(
+    record: Any,
+    *,
+    idx: int,
+    seen_paths: set[str],
+    seen_source_ids: set[str],
+) -> list[FieldViolation]:
+    prefix = f"records[{idx}]"
+    if not isinstance(record, dict):
+        return [FieldViolation(field=prefix, error="must be an object")]
+
+    violations: list[FieldViolation] = []
+    source_id = record.get("source_id")
+    if not _non_empty_string(source_id):
+        violations.append(FieldViolation(field=f"{prefix}.source_id", error="must be a non-empty string"))
+    else:
+        normalized_source_id = str(source_id).strip()
+        if normalized_source_id in seen_source_ids:
+            violations.append(
+                FieldViolation(field=f"{prefix}.source_id", error=f"duplicate source_id:{normalized_source_id}")
+            )
+        seen_source_ids.add(normalized_source_id)
+
+    path = record.get("path")
+    if not _non_empty_string(path):
+        violations.append(FieldViolation(field=f"{prefix}.path", error="must be a non-empty string"))
+    else:
+        normalized_path = str(path).strip()
+        if normalized_path in seen_paths:
+            violations.append(FieldViolation(field=f"{prefix}.path", error=f"duplicate path:{normalized_path}"))
+        seen_paths.add(normalized_path)
+
+    if not _valid_sha256(record.get("sha256")):
+        violations.append(FieldViolation(field=f"{prefix}.sha256", error="must be a lowercase sha256 hex digest"))
+
+    size = record.get("size_bytes")
+    if type(size) is not int or size <= 0:
+        violations.append(FieldViolation(field=f"{prefix}.size_bytes", error="must be a positive integer"))
+
+    for field in (
+        "source_title",
+        "publisher",
+        "source_type",
+        "raw_underlying_evidence_type",
+    ):
+        value = record.get(field)
+        if value is not None and not _non_empty_string(value):
+            violations.append(FieldViolation(field=f"{prefix}.{field}", error="must be a non-empty string"))
+    category_error = source_category_error(record.get("source_category"))
+    if category_error:
+        violations.append(FieldViolation(field=f"{prefix}.source_category", error=category_error))
+    retrieval_error = retrieval_source_type_error(record.get("retrieval_source_type"))
+    if retrieval_error:
+        violations.append(FieldViolation(field=f"{prefix}.retrieval_source_type", error=retrieval_error))
+    underlying_error = underlying_evidence_type_error(record.get("underlying_evidence_type"))
+    if underlying_error:
+        violations.append(FieldViolation(field=f"{prefix}.underlying_evidence_type", error=underlying_error))
+
+    return violations
+
+
+def _valid_sha256(value: Any) -> bool:
+    return isinstance(value, str) and bool(SHA256_RE.match(value.strip()))
+
+
+def _non_empty_string(value: Any) -> bool:
+    return isinstance(value, str) and bool(value.strip())

multi_agent_brief/contracts/schemas/source_item.py

@@ -0,0 +1,96 @@
+"""Contract for SourceItem."""
+
+from __future__ import annotations
+
+from typing import Any, ClassVar
+
+from multi_agent_brief.contracts.base import Contract, SchemaRegistry
+from multi_agent_brief.contracts.errors import FieldViolation
+from multi_agent_brief.contracts.source_metadata import (
+    VALID_RETRIEVAL_SOURCE_TYPES,
+    VALID_SOURCE_CATEGORIES,
+    VALID_UNDERLYING_EVIDENCE_TYPES,
+    retrieval_source_type_error,
+    source_category_error,
+    underlying_evidence_type_error,
+)
+
+REQUIRED_FIELDS = {"source_id", "source_name", "source_type", "title", "content"}
+OPTIONAL_FIELDS = {
+    "url",
+    "published_at",
+    "retrieved_at",
+    "language",
+    "reliability",
+    "dedupe_key",
+    "source_category",
+    "retrieval_source_type",
+    "underlying_evidence_type",
+    "raw_underlying_evidence_type",
+    "metadata",
+}
+ALL_FIELDS = REQUIRED_FIELDS | OPTIONAL_FIELDS
+
+
+@SchemaRegistry.register
+class SourceItemContract(Contract):
+    schema_id: ClassVar[str] = "source_item"
+    schema_version: ClassVar[str] = "v1"
+
+    @classmethod
+    def json_schema(cls) -> dict[str, Any]:
+        return {
+            "type": "object",
+            "required": sorted(REQUIRED_FIELDS),
+            "properties": {
+                "source_id": {"type": "string"},
+                "source_name": {"type": "string"},
+                "source_type": {"type": "string"},
+                "title": {"type": "string"},
+                "content": {"type": "string"},
+                "url": {"type": "string"},
+                "published_at": {"type": "string"},
+                "retrieved_at": {"type": "string"},
+                "language": {"type": "string"},
+                "reliability": {"type": "string", "enum": ["low", "medium", "high"]},
+                "dedupe_key": {"type": "string"},
+                "source_category": {"type": "string", "enum": sorted(VALID_SOURCE_CATEGORIES)},
+                "retrieval_source_type": {
+                    "type": "string",
+                    "enum": sorted(VALID_RETRIEVAL_SOURCE_TYPES),
+                },
+                "underlying_evidence_type": {
+                    "type": "string",
+                    "enum": sorted(VALID_UNDERLYING_EVIDENCE_TYPES),
+                },
+                "raw_underlying_evidence_type": {"type": "string"},
+                "metadata": {"type": "object"},
+            },
+            "additionalProperties": True,
+        }
+
+    @classmethod
+    def validate(cls, data: dict[str, Any]) -> list[FieldViolation]:
+        violations: list[FieldViolation] = []
+        for field in REQUIRED_FIELDS:
+            val = data.get(field)
+            if val is None or (isinstance(val, str) and not val.strip()):
+                violations.append(FieldViolation(field=field, error="required field is missing or blank"))
+        category_error = source_category_error(data.get("source_category"))
+        if category_error:
+            violations.append(FieldViolation(field="source_category", error=category_error))
+        retrieval_error = retrieval_source_type_error(data.get("retrieval_source_type"))
+        if retrieval_error:
+            violations.append(FieldViolation(field="retrieval_source_type", error=retrieval_error))
+        underlying_error = underlying_evidence_type_error(data.get("underlying_evidence_type"))
+        if underlying_error:
+            violations.append(FieldViolation(field="underlying_evidence_type", error=underlying_error))
+        # Unknown fields as warnings
+        unknown = set(data.keys()) - ALL_FIELDS
+        for field in sorted(unknown):
+            violations.append(FieldViolation(field=field, error="unknown field", severity="warning"))
+        return violations
+
+    @classmethod
+    def migrate(cls, data: dict[str, Any], from_version: str) -> dict[str, Any]:
+        return dict(data)  # no migration needed for v1

multi_agent_brief/contracts/semantic_assessment_status.py

@@ -0,0 +1,12 @@
+"""Shared Semantic Assessment Report status vocabulary."""
+
+from __future__ import annotations
+
+
+SEMANTIC_ASSESSMENT_REPORT_STATUSES = (
+    "not_available",
+    "valid",
+    "stale",
+    "missing_input",
+    "invalid_report",
+)

multi_agent_brief/contracts/source_metadata.py

@@ -0,0 +1,252 @@
+"""Shared source metadata validation helpers."""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Mapping
+from urllib.parse import urlparse
+
+SOURCE_CATEGORY_FIELD = "source_category"
+VALID_SOURCE_CATEGORIES = {
+    "clin" + "ical_registry",
+    "company_press_release",
+    "industry_database",
+    "market_report",
+    "news_media",
+    "other",
+    "peer_reviewed_paper",
+    "preprint",
+    "regulator",
+}
+VALID_RETRIEVAL_SOURCE_TYPES = {
+    "blog",
+    "company_pr",
+    "dataset",
+    "filing",
+    "local_file",
+    "news_media",
+    "other",
+    "paper_page",
+    "podcast",
+    "social",
+    "video",
+}
+VALID_UNDERLYING_EVIDENCE_TYPES = {
+    "company_claim",
+    "conference_demo",
+    "filing",
+    "interview",
+    "market_data",
+    "media_report",
+    "peer_reviewed_paper",
+    "regulator_record",
+    "unknown",
+}
+SOURCE_CATEGORY_ALIASES = {
+    "company_claim": "company_press_release",
+    "company_official": "company_press_release",
+    "company_pr": "company_press_release",
+    "company_release": "company_press_release",
+    "company_source": "company_press_release",
+    "data_provider": "industry_database",
+    "government_regulator": "regulator",
+    "industry_media": "news_media",
+    "industry_news": "news_media",
+    "industry_report": "market_report",
+    "market_data": "industry_database",
+    "market_data_provider": "industry_database",
+    "media": "news_media",
+    "media_report": "news_media",
+    "news": "news_media",
+    "official_regulator": "regulator",
+    "paper": "peer_reviewed_paper",
+    "regulator_record": "regulator",
+    "regulatory": "regulator",
+    "research_institution": "market_report",
+    "research_report": "market_report",
+    "trade_publication": "news_media",
+}
+RETRIEVAL_SOURCE_TYPE_ALIASES = {
+    "academic_paper": "paper_page",
+    "article": "news_media",
+    "cached": "local_file",
+    "cached_package": "local_file",
+    "company_blog": "blog",
+    "company_official": "company_pr",
+    "company_release": "company_pr",
+    "company_source": "company_pr",
+    "government_regulator": "other",
+    "industry_media": "news_media",
+    "industry_news": "news_media",
+    "local": "local_file",
+    "manual": "local_file",
+    "manual_url": "other",
+    "market_data": "dataset",
+    "market_data_provider": "dataset",
+    "media": "news_media",
+    "news": "news_media",
+    "official_regulator": "other",
+    "paper": "paper_page",
+    "preprint": "paper_page",
+    "regulator_record": "other",
+    "regulatory": "other",
+    "research_report": "paper_page",
+    "sec_filing": "filing",
+    "trade_publication": "news_media",
+}
+UNDERLYING_EVIDENCE_TYPE_ALIASES = {
+    "academic_paper": "peer_reviewed_paper",
+    "company_press_release": "company_claim",
+    "company_official": "company_claim",
+    "company_pr": "company_claim",
+    "company_release": "company_claim",
+    "company_source": "company_claim",
+    "data_provider": "market_data",
+    "government_regulator": "regulator_record",
+    "industry_database": "market_data",
+    "industry_media": "media_report",
+    "industry_news": "media_report",
+    "market_data_provider": "market_data",
+    "market_report": "market_data",
+    "media": "media_report",
+    "news": "media_report",
+    "news_media": "media_report",
+    "official_regulator": "regulator_record",
+    "paper": "peer_reviewed_paper",
+    "press_release": "company_claim",
+    "regulator": "regulator_record",
+    "regulatory": "regulator_record",
+    "research_institution": "market_data",
+    "research_report": "media_report",
+    "sec_filing": "filing",
+    "trade_publication": "media_report",
+}
+
+
+def non_empty_text(value: Any) -> bool:
+    return isinstance(value, str) and bool(value.strip())
+
+
+def source_url_error(value: Any) -> str | None:
+    if value is None or value == "":
+        return None
+    if not isinstance(value, str):
+        return "must be a string"
+    stripped = value.strip()
+    if not stripped:
+        return None
+    try:
+        parsed = urlparse(stripped)
+    except ValueError:
+        return "must be an http(s) URL"
+    if parsed.scheme not in {"http", "https"} or not parsed.netloc:
+        return "must be an http(s) URL"
+    return None
+
+
+def source_category_error(value: Any) -> str | None:
+    if value is None or value == "":
+        return None
+    if not isinstance(value, str):
+        return "must be a string"
+    if value.strip() not in VALID_SOURCE_CATEGORIES:
+        return f"must be one of {sorted(VALID_SOURCE_CATEGORIES)}"
+    return None
+
+
+def retrieval_source_type_error(value: Any) -> str | None:
+    if value is None or value == "":
+        return None
+    if not isinstance(value, str):
+        return "must be a string"
+    if value.strip() not in VALID_RETRIEVAL_SOURCE_TYPES:
+        return f"must be one of {sorted(VALID_RETRIEVAL_SOURCE_TYPES)}"
+    return None
+
+
+def underlying_evidence_type_error(value: Any) -> str | None:
+    if value is None or value == "":
+        return None
+    if not isinstance(value, str):
+        return "must be a string"
+    if value.strip() not in VALID_UNDERLYING_EVIDENCE_TYPES:
+        return f"must be one of {sorted(VALID_UNDERLYING_EVIDENCE_TYPES)}"
+    return None
+
+
+def normalize_source_category(*values: Any, default: str = "other") -> str:
+    for value in values:
+        normalized = _taxonomy_key(value)
+        if not normalized:
+            continue
+        if normalized in VALID_SOURCE_CATEGORIES:
+            return normalized
+        if normalized in SOURCE_CATEGORY_ALIASES:
+            return SOURCE_CATEGORY_ALIASES[normalized]
+    return default
+
+
+def normalize_retrieval_source_type(*values: Any, default: str = "other") -> str:
+    for value in values:
+        normalized = _taxonomy_key(value)
+        if not normalized:
+            continue
+        if normalized in VALID_RETRIEVAL_SOURCE_TYPES:
+            return normalized
+        if normalized in RETRIEVAL_SOURCE_TYPE_ALIASES:
+            return RETRIEVAL_SOURCE_TYPE_ALIASES[normalized]
+    return default
+
+
+def normalize_underlying_evidence_type(*values: Any, default: str = "unknown") -> str:
+    for value in values:
+        normalized = _taxonomy_key(value)
+        if not normalized:
+            continue
+        if normalized in VALID_UNDERLYING_EVIDENCE_TYPES:
+            return normalized
+        if normalized in UNDERLYING_EVIDENCE_TYPE_ALIASES:
+            return UNDERLYING_EVIDENCE_TYPE_ALIASES[normalized]
+    return default
+
+
+def _taxonomy_key(value: Any) -> str:
+    if not isinstance(value, str):
+        return ""
+    return re.sub(r"[^a-z0-9]+", "_", value.strip().lower()).strip("_")
+
+
+def has_valid_source_url(record: Mapping[str, Any]) -> bool:
+    value = record.get("source_url")
+    return non_empty_text(value) and source_url_error(value) is None
+
+
+def source_category_missing(record: Mapping[str, Any]) -> bool:
+    return not non_empty_text(record.get(SOURCE_CATEGORY_FIELD))
+
+
+def local_file_without_url_missing_identity(
+    record: Mapping[str, Any],
+    *,
+    default_source_type: str | None = None,
+) -> str | None:
+    source_type = record.get("source_type")
+    if isinstance(source_type, str):
+        normalized_source_type = source_type.strip() or default_source_type
+    elif source_type is None:
+        normalized_source_type = default_source_type
+    else:
+        return None
+    if not normalized_source_type:
+        return None
+    if normalized_source_type != "local_file":
+        return None
+    if has_valid_source_url(record):
+        return None
+    if source_category_missing(record):
+        return SOURCE_CATEGORY_FIELD
+    if not (
+        non_empty_text(record.get("source_title")) or non_empty_text(record.get("source_name"))
+    ):
+        return "source_title_or_source_name"
+    return None