plm-skill-kernel 1.0.0__py3-none-any.whl

plm_skill_kernel/skills/cleansing/dedupe.py

@@ -0,0 +1,162 @@
+"""``cleansing.dedupe`` / 1.0.0 / L2 — pure-function record dedupe.
+
+V1 dedupe primitives (no DB, no LLM):
+
+* **Exact**: record key matches verbatim → duplicates collapse.
+* **Normalised**: apply the same NFKC + whitespace collapse + case
+  fold pipeline as ``cleansing.normalise`` to the key, then compare.
+
+The skill picks one canonical representative per group (the FIRST
+occurrence; index order is preserved), reports the duplicate clusters,
+and returns a deduped list. No fuzzy / Levenshtein in V1 — that's a
+V1.1 expansion (would need a tunable threshold + a more expensive
+pairwise compute path that doesn't fit a pure synchronous skill body).
+
+Input contract (``payload``):
+
+    {
+        "records":  [str | dict, ...],         # required
+        "key":      str | None,                # required for dicts
+        "mode":     "exact" | "normalised",    # default "normalised"
+    }
+
+Output:
+
+    {
+        "deduped":     [str | dict, ...],
+        "duplicates":  [
+            {"key": <key>, "count": N, "indices": [i, j, ...]},
+            ...
+        ],
+        "stats":       {"records": N, "unique": K, "duplicates": D},
+    }
+
+Harvested from the dedupe pass that runs on imported product-structure
+nodes in ``services/cleansing_service`` (label-collision detection at
+import) — that backend path can migrate to invoking this skill after
+the Wave 2 Conv B carve.
+"""
+from __future__ import annotations
+
+import re
+import unicodedata
+from collections import OrderedDict
+from typing import Any, Dict, List, Optional
+
+from plm_skill_kernel import SkillContext, skill
+
+
+_WS_RE = re.compile(r"\s+")
+_VALID_MODES = frozenset({"exact", "normalised"})
+
+
+def _normalise_key(raw: Any) -> str:
+    """V1 normalised-key shape — same primitives as cleansing.normalise.
+
+    Kept inline rather than imported from the sibling module so the
+    skill is self-contained at import time (no cross-skill coupling
+    in the SDK registry).
+    """
+    if not isinstance(raw, str):
+        raw = str(raw)
+    val = unicodedata.normalize("NFKC", raw)
+    val = _WS_RE.sub(" ", val).strip().lower()
+    return val
+
+
+def _extract_key(
+    record: Any, key: Optional[str]
+) -> Any:
+    """Pull the dedup key from a record.
+
+    Strings dedupe by themselves; dicts dedupe by ``record[key]``.
+    ``None`` keys (missing field on a dict) are returned as-is and will
+    only collide with other ``None``-keyed records.
+    """
+    if isinstance(record, str):
+        return record
+    if isinstance(record, dict):
+        if key is None:
+            raise ValueError(
+                "cleansing.dedupe: payload['key'] is required when "
+                "records are dicts"
+            )
+        return record.get(key)
+    return record
+
+
+@skill(
+    id="cleansing.dedupe",
+    version="1.0.0",
+    description=(
+        "Pure-function dedupe — exact or normalised-key matching. "
+        "Picks the first occurrence as canonical; reports clusters."
+    ),
+    min_autonomy="L2",
+    streaming=False,
+    hitl_required=False,
+)
+async def cleansing_dedupe(
+    payload: Dict[str, Any], ctx: SkillContext
+) -> Dict[str, Any]:
+    """Dedupe a list of records by exact or normalised key."""
+    records = payload.get("records")
+    if not isinstance(records, list):
+        raise ValueError(
+            "cleansing.dedupe: payload['records'] must be a list"
+        )
+
+    mode = payload.get("mode", "normalised")
+    if mode not in _VALID_MODES:
+        raise ValueError(
+            f"cleansing.dedupe: payload['mode'] must be one of "
+            f"{sorted(_VALID_MODES)}; got {mode!r}"
+        )
+
+    key = payload.get("key")
+
+    # OrderedDict preserves first-seen order — that becomes the
+    # canonical pick-order (V1 spec: first occurrence wins).
+    canonical: "OrderedDict[Any, int]" = OrderedDict()
+    clusters: Dict[Any, List[int]] = {}
+
+    for idx, record in enumerate(records):
+        try:
+            raw_key = _extract_key(record, key)
+        except ValueError:
+            raise
+
+        cluster_key = (
+            _normalise_key(raw_key) if mode == "normalised" else raw_key
+        )
+
+        if cluster_key not in canonical:
+            canonical[cluster_key] = idx
+            clusters[cluster_key] = [idx]
+        else:
+            clusters[cluster_key].append(idx)
+
+    deduped = [records[i] for i in canonical.values()]
+    duplicate_clusters = [
+        {
+            "key": cluster_key,
+            "count": len(indices),
+            "indices": indices,
+        }
+        for cluster_key, indices in clusters.items()
+        if len(indices) > 1
+    ]
+    duplicate_total = sum(c["count"] - 1 for c in duplicate_clusters)
+
+    return {
+        "deduped": deduped,
+        "duplicates": duplicate_clusters,
+        "stats": {
+            "records": len(records),
+            "unique": len(deduped),
+            "duplicates": duplicate_total,
+        },
+    }
+
+
+__all__ = ["cleansing_dedupe"]

plm_skill_kernel/skills/cleansing/normalise.py

@@ -0,0 +1,178 @@
+"""``cleansing.normalise`` / 1.0.0 / L2 — pure-function record normaliser.
+
+V1 normalisation primitives (idempotent, no side effects):
+
+* Unicode NFKC normalisation — collapses compatibility forms.
+* Whitespace collapse — strip + replace runs of whitespace with a
+  single space.
+* Optional case fold — caller passes ``case="lower"`` / ``"upper"``
+  / ``"none"``; default is ``"lower"``.
+* Optional punctuation strip — caller passes ``strip_punctuation=True``
+  to remove ``.,;:!?-_/\\`` and matching brackets; default ``False``.
+
+Input contract (``payload``):
+
+    {
+        "records":            [str | dict, ...],   # required
+        "fields":             [str, ...] | None,    # if records=dicts
+        "case":               "lower" | "upper" | "none",  # default "lower"
+        "strip_punctuation":  bool,                 # default False
+    }
+
+Output:
+
+    {
+        "normalised":         [str | dict, ...],   # mirrors input shape
+        "stats":              {"records": N, "changed": K},
+    }
+
+The skill is harvested from the CSV-mining normalisation step in
+``agents/process_generator`` + the label cleansing applied in
+``services/cleansing_service``. Both call sites can migrate to the
+Skill once the V1.1 HTTP loopback lands (Wave 2 Conv B).
+"""
+from __future__ import annotations
+
+import re
+import unicodedata
+from typing import Any, Dict, List, Optional, Tuple
+
+from plm_skill_kernel import SkillContext, skill
+
+
+# Punctuation set per V1 spec — compact regex compiled once per process.
+_PUNCT_RE = re.compile(r"[.,;:!?\-_/\\\(\)\[\]\{\}]+")
+_WS_RE = re.compile(r"\s+")
+
+
+_VALID_CASES = frozenset({"lower", "upper", "none"})
+
+
+def _normalise_string(
+    raw: str, *, case: str, strip_punctuation: bool
+) -> str:
+    """Apply the V1 normalisation pipeline to a single string."""
+    if not isinstance(raw, str):
+        raw = str(raw)
+    # Step 1 — Unicode NFKC. Collapses compat-forms (full-width digits,
+    # ligatures, etc.) into their canonical equivalents.
+    val = unicodedata.normalize("NFKC", raw)
+    # Step 2 — punctuation (optional, before whitespace collapse so
+    # punctuation-induced spacing collapses cleanly).
+    if strip_punctuation:
+        val = _PUNCT_RE.sub(" ", val)
+    # Step 3 — whitespace collapse.
+    val = _WS_RE.sub(" ", val).strip()
+    # Step 4 — case fold (default lower for downstream-comparison
+    # stability; punctuation/whitespace-only inputs collapse to "").
+    if case == "lower":
+        val = val.lower()
+    elif case == "upper":
+        val = val.upper()
+    return val
+
+
+def _normalise_dict(
+    record: Dict[str, Any],
+    *,
+    fields: Optional[List[str]],
+    case: str,
+    strip_punctuation: bool,
+) -> Tuple[Dict[str, Any], bool]:
+    """Apply normalisation to selected fields; non-selected fields pass through."""
+    out: Dict[str, Any] = dict(record)
+    changed = False
+    target_fields = fields if fields is not None else list(record.keys())
+    for field in target_fields:
+        if field not in out:
+            continue
+        original = out[field]
+        if not isinstance(original, str):
+            continue
+        normalised = _normalise_string(
+            original, case=case, strip_punctuation=strip_punctuation
+        )
+        if normalised != original:
+            changed = True
+            out[field] = normalised
+    return out, changed
+
+
+@skill(
+    id="cleansing.normalise",
+    version="1.0.0",
+    description=(
+        "Pure-function normalisation of records (Unicode NFKC, "
+        "whitespace collapse, case fold, optional punctuation strip)."
+    ),
+    min_autonomy="L2",
+    streaming=False,
+    hitl_required=False,
+)
+async def cleansing_normalise(
+    payload: Dict[str, Any], ctx: SkillContext
+) -> Dict[str, Any]:
+    """Normalise a list of records.
+
+    Records can be plain strings or dicts. When dicts, the optional
+    ``fields`` array selects which keys to normalise; default = all
+    string-valued keys.
+    """
+    records = payload.get("records")
+    if not isinstance(records, list):
+        raise ValueError(
+            "cleansing.normalise: payload['records'] must be a list"
+        )
+    fields = payload.get("fields")
+    if fields is not None and not isinstance(fields, list):
+        raise ValueError(
+            "cleansing.normalise: payload['fields'] must be a list when set"
+        )
+
+    case_raw = payload.get("case", "lower")
+    if case_raw not in _VALID_CASES:
+        raise ValueError(
+            f"cleansing.normalise: payload['case'] must be one of "
+            f"{sorted(_VALID_CASES)}; got {case_raw!r}"
+        )
+    case: str = case_raw
+
+    strip_punctuation = bool(payload.get("strip_punctuation", False))
+
+    normalised: List[Any] = []
+    changed_count = 0
+    for record in records:
+        if isinstance(record, str):
+            new_val = _normalise_string(
+                record, case=case, strip_punctuation=strip_punctuation
+            )
+            if new_val != record:
+                changed_count += 1
+            normalised.append(new_val)
+        elif isinstance(record, dict):
+            new_record, changed = _normalise_dict(
+                record,
+                fields=fields,
+                case=case,
+                strip_punctuation=strip_punctuation,
+            )
+            if changed:
+                changed_count += 1
+            normalised.append(new_record)
+        else:
+            # Unknown shape — pass through untouched. V1 contract is
+            # str-or-dict; arbitrary shapes (numbers, None) flow
+            # without normalisation rather than raising, so callers
+            # can pass mixed-shape lists without pre-filtering.
+            normalised.append(record)
+
+    return {
+        "normalised": normalised,
+        "stats": {
+            "records": len(records),
+            "changed": changed_count,
+        },
+    }
+
+
+__all__ = ["cleansing_normalise"]

plm_skill_kernel/skills/documents/__init__.py

@@ -0,0 +1,12 @@
+"""Wave 3 Conv A — ``documents.*`` skill namespace.
+
+Subpackages:
+
+* ``documents.parse`` — thin proxy to ``services.agent_document_handler``
+  (W2-CARVE-3 façade). L1 — classifies + extracts content from caller-
+  supplied attachments without persisting state.
+
+The ``__init__.py`` is intentionally empty — skills register themselves at
+import time via the ``@skill`` decorator; the boot fan-out lives in
+:mod:`plm_skill_kernel.registry`.
+"""

plm_skill_kernel/skills/documents/parse.py

@@ -0,0 +1,202 @@
+"""``documents.parse`` / 1.0.0 / L1 — thin proxy to the W2-CARVE-3 façade.
+
+Q-W3A-2 (Wave 3 Conv A) picked the thin-proxy shape: the skill imports +
+delegates to ``services.agent_document_handler`` (the Wave 2 Conv C
+W2-CARVE-3 façade that froze the V1 input shape) at invocation time.
+Mirrors the ``bpmn.generate`` Decision #62 precedent — the V1 dispatch
+boundary is a lazy-import contract; V1.1 will replace it with an MCP-
+bound out-of-process call.
+
+V1 contract:
+
+    Input payload:
+        {
+            "filename":      str,                      # required
+            "content_type":  str | None,               # optional
+            "pdf_bytes_b64": str | None,               # optional, base64
+            "text":          str | None,               # optional
+        }
+
+        At least one of ``pdf_bytes_b64`` / ``text`` MUST be present
+        unless the caller only wants classification metadata.
+
+    Output:
+        {
+            "filename":      str,
+            "kind":          "pdf" | "text" | "viewer_3d"
+                             | "binary" | "unknown",
+            "extension":     str,
+            "use_3d_viewer": bool,
+            "content":       str,        # extracted text (or "" when N/A)
+            "extracted":     bool,       # True if PDF/text payload yielded content
+        }
+
+L1 (suggest-only) — the skill returns the parse result for the caller
+to use; it does NOT persist or commit. The caller is responsible for
+attaching the parse result to a CleansingDocument or Workbench upload.
+
+Architecture note: ``services.agent_document_handler`` is a backend
+module (``02_App/backend/services/``) NOT under ``plm_skill_kernel`` or
+``plm_engine_core``. Importing it here creates a runtime dependency on
+the backend Python path — the import is therefore done LAZILY inside
+the function body so:
+
+  1. The plm-skill-kernel package itself can be imported without
+     requiring the backend on PYTHONPATH (e.g. a clean container image
+     of the kernel-only service).
+  2. Tests that don't exercise document parsing don't pay the import
+     cost (and don't require the backend deps to be installed).
+
+The lazy-import pattern is the V1 dispatch-boundary contract; V1.1 will
+replace it with an MCP-bound out-of-process call once the backend's
+parse entry point itself migrates to an MCP server.
+"""
+from __future__ import annotations
+
+import base64
+import binascii
+import logging
+import sys
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+from plm_skill_kernel import SkillContext, skill
+
+
+logger = logging.getLogger(__name__)
+
+
+def _ensure_backend_on_path() -> None:
+    """Best-effort: add ``02_App/backend`` to ``sys.path`` if it isn't.
+
+    Same pattern as :mod:`plm_skill_kernel.skills.bpmn.generate` — kept
+    duplicated rather than extracted to a helper so each skill stays
+    self-contained at import time (no cross-skill coupling at the SDK
+    registry layer).
+    """
+    here = Path(__file__).resolve()
+    # plm_skill_kernel/skills/documents/parse.py → 4× parents = plm-skill-kernel/
+    # 5× parents = 02_App/.
+    candidate = here.parents[4] / "backend"
+    if candidate.exists():
+        candidate_str = str(candidate)
+        if candidate_str not in sys.path:
+            sys.path.insert(0, candidate_str)
+
+
+def _decode_pdf_bytes(b64: str) -> bytes:
+    """Decode a base64-encoded PDF payload.
+
+    Raises ValueError on malformed input — V1 contract surfaces the
+    error as a structured envelope at the dispatcher boundary.
+    """
+    try:
+        return base64.b64decode(b64, validate=True)
+    except (binascii.Error, ValueError) as exc:
+        raise ValueError(
+            f"documents.parse: payload['pdf_bytes_b64'] is not valid base64 ({exc})"
+        ) from exc
+
+
+@skill(
+    id="documents.parse",
+    version="1.0.0",
+    description=(
+        "Classify + extract content from a caller-supplied attachment "
+        "(PDF text extraction, 3D-viewer hint, text passthrough). "
+        "Suggest-only (L1) — caller persists the result. Wraps the "
+        "Wave 2 Conv C W2-CARVE-3 façade."
+    ),
+    min_autonomy="L1",
+    streaming=False,
+    hitl_required=False,
+)
+async def documents_parse(
+    payload: Dict[str, Any], ctx: SkillContext
+) -> Dict[str, Any]:
+    """Thin proxy to ``services.agent_document_handler``.
+
+    Validates the V1 contract surface (``filename`` is required +
+    ``pdf_bytes_b64`` / ``text`` shapes are validated when present)
+    before reaching the backend; once the proxy fires, the W2-CARVE-3
+    façade owns classification + extraction.
+    """
+    filename = payload.get("filename")
+    if not isinstance(filename, str) or not filename:
+        raise ValueError(
+            "documents.parse: payload['filename'] must be a non-empty string"
+        )
+
+    content_type_raw = payload.get("content_type")
+    if content_type_raw is not None and not isinstance(content_type_raw, str):
+        raise ValueError(
+            "documents.parse: payload['content_type'] must be a string when set"
+        )
+    content_type: Optional[str] = content_type_raw
+
+    pdf_b64 = payload.get("pdf_bytes_b64")
+    if pdf_b64 is not None and not isinstance(pdf_b64, str):
+        raise ValueError(
+            "documents.parse: payload['pdf_bytes_b64'] must be a string when set"
+        )
+
+    text_payload = payload.get("text")
+    if text_payload is not None and not isinstance(text_payload, str):
+        raise ValueError(
+            "documents.parse: payload['text'] must be a string when set"
+        )
+
+    _ensure_backend_on_path()
+    try:
+        from studio_backend.services.agent_document_handler import (  # type: ignore[import-not-found]
+            AttachmentKind,
+            classify_attachment,
+            extract_pdf_text,
+        )
+    except ImportError as exc:
+        raise RuntimeError(
+            "documents.parse: backend module `services.agent_document_handler` "
+            "is not importable from this kernel process. The V1 thin-"
+            "proxy shape requires the backend on PYTHONPATH; install "
+            "the backend editable or run via 02_App/run.py."
+        ) from exc
+
+    descriptor = classify_attachment(filename, content_type=content_type)
+
+    logger.info(
+        "documents.parse dispatching filename=%s kind=%s tenant=%s run_id=%s",
+        filename,
+        descriptor.kind.value,
+        ctx.tenant_id,
+        ctx.run_id,
+    )
+
+    extracted = False
+    content_out = ""
+
+    # PDF path — decode + delegate to the W2-CARVE-3 façade.
+    if descriptor.kind == AttachmentKind.PDF and pdf_b64 is not None:
+        pdf_bytes = _decode_pdf_bytes(pdf_b64)
+        content_out = extract_pdf_text(pdf_bytes) or ""
+        extracted = bool(content_out)
+
+    # Text passthrough path — caller already supplied the content.
+    elif descriptor.kind == AttachmentKind.TEXT and text_payload is not None:
+        content_out = text_payload
+        extracted = bool(content_out)
+
+    # Viewer-3D / binary / unknown / classification-only path — no
+    # extraction, return descriptor metadata so the caller can decide
+    # whether to render a 3D viewer pane or fall back to a download.
+
+    return {
+        "filename": descriptor.filename,
+        "kind": descriptor.kind.value,
+        "extension": descriptor.extension,
+        "use_3d_viewer": descriptor.use_3d_viewer,
+        "content": content_out,
+        "extracted": extracted,
+    }
+
+
+__all__ = ["documents_parse"]

plm_skill_kernel/skills/knowledge/__init__.py

@@ -0,0 +1,32 @@
+"""FTR-603 Wave 5 Conv C — Knowledge Services V1 primitive skills.
+
+The 5 active V1 primitives (D-LOCKED-23 / PRD §8.11.3) are exposed here
+as kernel-routable skills under the ``knowledge.*`` namespace. The 2
+deferred primitives (``explain`` / ``context_pack``) ship as YAML
+``draft`` capabilities only — they do NOT register a kernel skill in V1
+because their dispatch returns ``KS_PRIMITIVE_DEFERRED_V1_1`` at the
+envelope layer (engine-core), not at the kernel layer.
+
+V1 thin-proxy contract (Decision #122 / Q-W5C-1): each skill validates
+its primitive request via the matching ``plm_shared.knowledge.primitives_v1``
+schema and produces a structured response dict carrying:
+
+    {
+        "payload":           <primitive response payload, dict>,
+        "source_refs":       [<KnowledgeSourceRef as dict>, ...],
+        "evidence_refs":     [<KnowledgeSourceRef as dict>, ...],
+        "connector_access":  [<ConnectorAccessEntry as dict>, ...],
+        "baseline_version":  <str>,
+    }
+
+The envelope wrapping (engine-core ``KnowledgeServiceEnvelope.wrap``)
+consumes this shape and produces the wire-shape ``KnowledgeEnvelopeV1``
+on the Core side at dispatch time. Keeping the envelope wrapping on the
+Core side preserves the import-linter contract
+``plm_skill_kernel-must-not-depend-on-plm_engine_core``.
+
+Module layout per Decision #124 / Q-W5C-3 — one module per primitive.
+"""
+from __future__ import annotations
+
+__all__: list[str] = []