eq-layer 0.1.0__py3-none-any.whl

eq_layer/dbdc3.py

@@ -0,0 +1,197 @@
+from __future__ import annotations
+
+import hashlib
+import io
+import json
+import urllib.request
+import zipfile
+from collections import Counter
+from dataclasses import dataclass
+
+DBDC3_URL = "https://dbd-challenge.github.io/dbdc3/data/DBDC3.zip"
+DBDC3_ARCHIVE_SHA256 = "736229795dc3732f8e6bb421f094dc820ef944fef9d9d320d4110ef992b60e85"
+
+
+@dataclass(frozen=True)
+class BreakdownTurn:
+    source_file: str
+    split: str
+    dialogue_id: str
+    turn_index: int
+    text: str
+    context: tuple[str, ...]
+    counts: dict[str, int]
+
+    @property
+    def model_text(self) -> str:
+        context = "\n".join(self.context)
+        if context:
+            return f"{context}\n[SYSTEM]\n{self.text}"
+        return self.text
+
+    @property
+    def score(self) -> float:
+        total = sum(self.counts.values())
+        if not total:
+            return 0.0
+        weighted = (
+            self.counts.get("T", 0) * 0.5
+            + self.counts.get("X", 0) * 1.0
+        )
+        return weighted / total
+
+    @property
+    def hard_breakdown(self) -> bool:
+        total = sum(self.counts.values())
+        if not total:
+            return False
+        return self.counts.get("X", 0) / total >= 0.5
+
+
+def download_archive() -> bytes:
+    request = urllib.request.Request(
+        DBDC3_URL,
+        headers={"User-Agent": "eq-layer-dbdc3-audit/0.1"},
+    )
+    with urllib.request.urlopen(request, timeout=120) as response:
+        return response.read()
+
+
+def sha256_bytes(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def _iter_json_members(archive: zipfile.ZipFile) -> list[str]:
+    return sorted(
+        name
+        for name in archive.namelist()
+        if name.lower().endswith(".json")
+    )
+
+
+def _english_member(name: str) -> bool:
+    low = name.lower().replace("\\", "/")
+    return "/en/" in f"/{low}"
+
+
+def _effective_english_members(names: list[str]) -> list[str]:
+    english = [name for name in names if _english_member(name)]
+    revised = [
+        name
+        for name in english
+        if "/dbdc3_revised/en/" in name.lower().replace("\\", "/")
+    ]
+    if revised:
+        return revised
+    return [
+        name
+        for name in english
+        if "/dbdc3/en/" in name.lower().replace("\\", "/")
+    ]
+
+
+def _split_from_member(name: str) -> str:
+    low = name.lower().replace("\\", "/")
+    if "/en/dev/" in low:
+        return "dev"
+    if "/en/eval/" in low or "/en/test/" in low:
+        return "eval"
+    return "unknown"
+
+
+def _dialogue_id(payload: dict, member: str) -> str:
+    for key in ("dialogue-id", "dialogue_id", "dialog-id", "id"):
+        if payload.get(key) is not None:
+            return str(payload[key])
+    return member.rsplit("/", 1)[-1].rsplit(".", 1)[0]
+
+
+def parse_english_turns(data: bytes) -> tuple[list[BreakdownTurn], dict]:
+    turns: list[BreakdownTurn] = []
+    with zipfile.ZipFile(io.BytesIO(data)) as archive:
+        members = _iter_json_members(archive)
+        all_english = [name for name in members if _english_member(name)]
+        english = _effective_english_members(members)
+
+        for member in english:
+            with archive.open(member) as fh:
+                payload = json.load(fh)
+
+            dialogues = payload if isinstance(payload, list) else [payload]
+            for dialogue in dialogues:
+                if not isinstance(dialogue, dict):
+                    continue
+                dialogue_turns = dialogue.get("turns")
+                if not isinstance(dialogue_turns, list):
+                    continue
+
+                history: list[str] = []
+                did = _dialogue_id(dialogue, member)
+                for position, turn in enumerate(dialogue_turns):
+                    if not isinstance(turn, dict):
+                        continue
+                    text = str(turn.get("utterance") or "").strip()
+                    speaker = str(turn.get("speaker") or "").upper()
+                    annotations = turn.get("annotations") or []
+
+                    if speaker == "S" and text:
+                        counts = Counter()
+                        for ann in annotations:
+                            if not isinstance(ann, dict):
+                                continue
+                            label = str(ann.get("breakdown") or "").upper()
+                            if label in {"O", "T", "X"}:
+                                counts[label] += 1
+                        if counts:
+                            turns.append(
+                                BreakdownTurn(
+                                    source_file=member,
+                                    split=_split_from_member(member),
+                                    dialogue_id=did,
+                                    turn_index=int(turn.get("turn-index", position)),
+                                    text=text,
+                                    context=tuple(history[-3:]),
+                                    counts=dict(counts),
+                                )
+                            )
+
+                    if text:
+                        history.append(f"{speaker}: {text}")
+
+        meta = {
+            "zip_members": len(members),
+            "all_english_json_members": len(all_english),
+            "effective_english_json_members": len(english),
+            "effective_source": (
+                "dbdc3_revised"
+                if any("/dbdc3_revised/en/" in name.lower().replace("\\", "/") for name in english)
+                else "dbdc3"
+            ),
+            "effective_dev_members": sum("/en/dev/" in name.lower().replace("\\", "/") for name in english),
+            "effective_eval_members": sum(
+                "/en/eval/" in name.lower().replace("\\", "/")
+                or "/en/test/" in name.lower().replace("\\", "/")
+                for name in english
+            ),
+        }
+    return turns, meta
+
+
+def inventory(turns: list[BreakdownTurn]) -> dict:
+    aggregate = Counter()
+    hard = 0
+    for turn in turns:
+        aggregate.update(turn.counts)
+        hard += int(turn.hard_breakdown)
+
+    return {
+        "system_turns": len(turns),
+        "dialogues": len({turn.dialogue_id for turn in turns}),
+        "annotation_counts": dict(sorted(aggregate.items())),
+        "hard_breakdown_turns": hard,
+        "hard_breakdown_rate": round(hard / len(turns), 4) if turns else 0.0,
+        "mean_breakdown_score": round(
+            sum(turn.score for turn in turns) / len(turns),
+            4,
+        ) if turns else 0.0,
+    }

eq_layer/dialogbank_iso.py

@@ -0,0 +1,272 @@
+from __future__ import annotations
+
+import html.parser
+import re
+import urllib.parse
+import urllib.request
+import xml.etree.ElementTree as ET
+from collections import Counter
+from dataclasses import dataclass
+
+DIALOGBANK_BASE = "https://dialogbank.lsv.uni-saarland.de/"
+DIALOGBANK_ENGLISH_PAGES = (
+    "https://dialogbank.lsv.uni-saarland.de/?page_id=90",   # HCRC MapTask
+    "https://dialogbank.lsv.uni-saarland.de/?page_id=92",   # Switchboard
+    "https://dialogbank.lsv.uni-saarland.de/?page_id=312",  # DBOX
+    "https://dialogbank.lsv.uni-saarland.de/?page_id=280",  # TRAINS
+)
+
+XML_ID = "{http://www.w3.org/XML/1998/namespace}id"
+
+
+class _LinkParser(html.parser.HTMLParser):
+    def __init__(self) -> None:
+        super().__init__()
+        self.links: list[str] = []
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        if tag != "a":
+            return
+        for key, value in attrs:
+            if key == "href" and value:
+                self.links.append(value)
+
+
+def fetch_text(url: str) -> str:
+    request = urllib.request.Request(
+        url,
+        headers={"User-Agent": "eq-layer-dialogbank-audit/0.1"},
+    )
+    with urllib.request.urlopen(request, timeout=30) as response:
+        return response.read().decode("utf-8")
+
+
+def discover_diaml_urls() -> list[str]:
+    urls: set[str] = set()
+    for page in DIALOGBANK_ENGLISH_PAGES:
+        parser = _LinkParser()
+        parser.feed(fetch_text(page))
+        for href in parser.links:
+            absolute = urllib.parse.urljoin(DIALOGBANK_BASE, href)
+            if absolute.lower().endswith(".diaml"):
+                urls.add(absolute)
+    return sorted(urls)
+
+
+@dataclass(frozen=True)
+class ISOExample:
+    dialogue: str
+    text: str
+    function: str
+    dimension: str
+
+
+def _local(tag: str) -> str:
+    return tag.rsplit("}", 1)[-1]
+
+
+def _strip_ref(value: str | None) -> str:
+    return (value or "").lstrip("#")
+
+
+def _repair_xml_text(xml_text: str) -> str:
+    # Some legacy DialogBank files contain bare ampersands or control
+    # characters in transcribed speech. Repair only characters that XML 1.0
+    # cannot represent; do not alter annotation structure or labels.
+    xml_text = re.sub(
+        r"&(?!#\d+;|#x[0-9A-Fa-f]+;|[A-Za-z_][A-Za-z0-9_.:-]*;)",
+        "&",
+        xml_text,
+    )
+    return "".join(
+        ch
+        for ch in xml_text
+        if ch in "\t\n\r" or ord(ch) >= 0x20
+    )
+
+
+
+def _parse_attrs(raw: str) -> dict[str, str]:
+    attrs: dict[str, str] = {}
+    for key, _quote, value in re.findall(
+        r"([A-Za-z_][\w:.-]*)\s*=\s*(['\"])(.*?)\2",
+        raw,
+        flags=re.DOTALL,
+    ):
+        attrs[key] = html.unescape(value)
+    return attrs
+
+
+def _plain_xml_text(raw: str) -> str:
+    without_tags = re.sub(r"<[^>]+>", "", raw)
+    return " ".join(html.unescape(without_tags).split())
+
+
+def _parse_diaml_legacy(xml_text: str, source_url: str) -> list[ISOExample]:
+    """Extract the annotation graph without requiring globally valid XML.
+
+    A few legacy DialogBank files contain malformed transcript characters.
+    The annotation tags themselves are regular enough to recover the fields
+    EQ-Layer needs. This fallback never rewrites or fabricates labels.
+    """
+    words: dict[str, str] = {}
+    for match in re.finditer(r"<(?:[A-Za-z_][\w.-]*:)?w\b([^>]*)>(.*?)</(?:[A-Za-z_][\w.-]*:)?w>", xml_text, flags=re.DOTALL):
+        attrs = _parse_attrs(match.group(1))
+        wid = attrs.get("xml:id")
+        if wid:
+            words[wid] = _plain_xml_text(match.group(2))
+
+    verbal_segments: dict[str, list[str]] = {}
+    for match in re.finditer(
+        r"<(?:[A-Za-z_][\w.-]*:)?spanGrp\b([^>]*)>(.*?)</(?:[A-Za-z_][\w.-]*:)?spanGrp>",
+        xml_text,
+        flags=re.DOTALL,
+    ):
+        attrs = _parse_attrs(match.group(1))
+        if attrs.get("type") != "functionalVerbalSegment":
+            continue
+        sid = attrs.get("xml:id")
+        if not sid:
+            continue
+        ids: list[str] = []
+        for span in re.finditer(r"<(?:[A-Za-z_][\w.-]*:)?span\b([^>]*)/?>", match.group(2)):
+            span_attrs = _parse_attrs(span.group(1))
+            start = _strip_ref(span_attrs.get("from"))
+            end = _strip_ref(span_attrs.get("to"))
+            if start:
+                ids.append(start)
+            if end and end != start:
+                ids.append(end)
+        verbal_segments[sid] = ids
+
+    functional_to_verbal: dict[str, str] = {}
+    for match in re.finditer(r"<(?:[A-Za-z_][\w.-]*:)?fs\b([^>]*)>(.*?)</(?:[A-Za-z_][\w.-]*:)?fs>", xml_text, flags=re.DOTALL):
+        attrs = _parse_attrs(match.group(1))
+        if attrs.get("type") != "functionalSegment":
+            continue
+        fsid = attrs.get("xml:id")
+        if not fsid:
+            continue
+        for fmatch in re.finditer(r"<(?:[A-Za-z_][\w.-]*:)?f\b([^>]*)/?>", match.group(2)):
+            fattrs = _parse_attrs(fmatch.group(1))
+            if fattrs.get("name") == "verbalComponent":
+                functional_to_verbal[fsid] = _strip_ref(fattrs.get("fVal"))
+                break
+
+    examples: list[ISOExample] = []
+    for match in re.finditer(r"<(?:[A-Za-z_][\w.-]*:)?dialogueAct\b([^>]*)/?>", xml_text):
+        attrs = _parse_attrs(match.group(1))
+        function = attrs.get("communicativeFunction")
+        target = _strip_ref(attrs.get("target"))
+        if not function or not target:
+            continue
+        verbal = functional_to_verbal.get(target)
+        token_ids = verbal_segments.get(verbal or "", [])
+        text = " ".join(words.get(token, "") for token in token_ids).strip()
+        if not text:
+            continue
+        examples.append(
+            ISOExample(
+                dialogue=source_url,
+                text=text,
+                function=function,
+                dimension=attrs.get("dimension", ""),
+            )
+        )
+
+    if not examples:
+        tags = Counter(
+            re.findall(r"<\s*/?\s*([A-Za-z_][\w:.-]*)", xml_text)
+        ).most_common(24)
+        raise ValueError(
+            f"Legacy DialogBank parser recovered no dialogue acts: {source_url}; "
+            f"top_tags={tags}"
+        )
+    return examples
+
+
+def parse_diaml(xml_text: str, source_url: str) -> list[ISOExample]:
+    try:
+        root = ET.fromstring(xml_text)
+    except ET.ParseError:
+        try:
+            root = ET.fromstring(_repair_xml_text(xml_text))
+        except ET.ParseError:
+            return _parse_diaml_legacy(xml_text, source_url)
+
+    words: dict[str, str] = {}
+    for elem in root.iter():
+        if _local(elem.tag) != "w":
+            continue
+        wid = elem.attrib.get(XML_ID)
+        if wid:
+            words[wid] = "".join(elem.itertext()).strip()
+
+    verbal_segments: dict[str, list[str]] = {}
+    for elem in root.iter():
+        if _local(elem.tag) != "spanGrp" or elem.attrib.get("type") != "functionalVerbalSegment":
+            continue
+        sid = elem.attrib.get(XML_ID)
+        if not sid:
+            continue
+        ids = []
+        for span in elem:
+            if _local(span.tag) != "span":
+                continue
+            start = _strip_ref(span.attrib.get("from"))
+            end = _strip_ref(span.attrib.get("to"))
+            if start:
+                ids.append(start)
+            if end and end != start:
+                ids.append(end)
+        verbal_segments[sid] = ids
+
+    functional_to_verbal: dict[str, str] = {}
+    for elem in root.iter():
+        if _local(elem.tag) != "fs" or elem.attrib.get("type") != "functionalSegment":
+            continue
+        fsid = elem.attrib.get(XML_ID)
+        if not fsid:
+            continue
+        for child in elem:
+            if _local(child.tag) == "f" and child.attrib.get("name") == "verbalComponent":
+                functional_to_verbal[fsid] = _strip_ref(child.attrib.get("fVal"))
+
+    examples: list[ISOExample] = []
+    for elem in root.iter():
+        if _local(elem.tag) != "dialogueAct":
+            continue
+        function = elem.attrib.get("communicativeFunction")
+        target = _strip_ref(elem.attrib.get("target"))
+        if not function or not target:
+            continue
+        verbal = functional_to_verbal.get(target)
+        token_ids = verbal_segments.get(verbal or "", [])
+        text = " ".join(words.get(token, "") for token in token_ids).strip()
+        if not text:
+            continue
+        examples.append(
+            ISOExample(
+                dialogue=source_url,
+                text=text,
+                function=function,
+                dimension=elem.attrib.get("dimension", ""),
+            )
+        )
+    return examples
+
+
+def load_dialogbank_english() -> list[ISOExample]:
+    examples: list[ISOExample] = []
+    for url in discover_diaml_urls():
+        examples.extend(parse_diaml(fetch_text(url), url))
+    return examples
+
+
+def inventory(examples: list[ISOExample]) -> dict:
+    counts = Counter(example.function for example in examples)
+    return {
+        "dialogues": len({example.dialogue for example in examples}),
+        "examples": len(examples),
+        "functions": dict(sorted(counts.items())),
+    }