sofic 0.1.0__py3-none-any.whl

sofic/automata/papni.py

@@ -0,0 +1,301 @@
+"""Passive visibly-pushdown topology learning via PAPNI preprocessing and RPNI."""
+
+from __future__ import annotations
+
+from collections.abc import Hashable, Sequence
+from dataclasses import dataclass
+from typing import Any
+
+from sofic.automata.dfa import DFA
+from sofic.automata.rpni import learn_dfa_rpni
+from sofic.graph import ATTR_KIND, ATTR_SYMBOL, KIND_CALL, KIND_INTERNAL, KIND_RETURN
+from sofic.shifts.sofic_dyck import SoficDyckShift, TransitionRef, transition_ref
+
+__all__ = [
+    "DyckAlphabet",
+    "is_well_matched",
+    "learn_sofic_dyck_shift_papni",
+    "papni_encode",
+    "papni_encode_samples",
+    "sofic_dyck_shift_from_papni_dfa",
+]
+
+
+@dataclass(frozen=True, slots=True)
+class DyckAlphabet:
+    """Partition of symbols into call, return, and internal roles."""
+
+    call_alphabet: frozenset[Any]
+    return_alphabet: frozenset[Any]
+    internal_alphabet: frozenset[Any]
+
+    def __post_init__(self) -> None:
+        union = self.call_alphabet | self.return_alphabet | self.internal_alphabet
+        if len(union) != len(self.call_alphabet) + len(self.return_alphabet) + len(self.internal_alphabet):
+            raise ValueError("call, return, and internal alphabets must be disjoint")
+
+    @property
+    def symbol_alphabet(self) -> frozenset[Any]:
+        return self.call_alphabet | self.return_alphabet | self.internal_alphabet
+
+    def classify(self, symbol: Any) -> str:
+        if symbol in self.call_alphabet:
+            return KIND_CALL
+        if symbol in self.return_alphabet:
+            return KIND_RETURN
+        if symbol in self.internal_alphabet:
+            return KIND_INTERNAL
+        raise ValueError(f"symbol {symbol!r} not in Dyck alphabet")
+
+
+def is_well_matched(word: Sequence[Any], alphabet: DyckAlphabet) -> bool:
+    """Return whether ``word`` is well-matched under a symbol counter (PAPNI Alg. 1)."""
+    counter = 0
+    for symbol in word:
+        if symbol in alphabet.call_alphabet:
+            counter += 1
+        elif symbol in alphabet.return_alphabet:
+            counter -= 1
+            if counter < 0:
+                return False
+        elif symbol not in alphabet.internal_alphabet:
+            return False
+    return counter == 0
+
+
+def papni_encode(word: Sequence[Any], alphabet: DyckAlphabet) -> tuple[Any, ...]:
+    """Convert a well-matched word to its stack-aware representation (PAPNI Alg. 2)."""
+    if not is_well_matched(word, alphabet):
+        raise ValueError("word is not well-matched")
+
+    encoded: list[Any] = []
+    stack: list[Any] = []
+    for symbol in word:
+        if symbol in alphabet.call_alphabet:
+            stack.append(symbol)
+            encoded.append(symbol)
+        elif symbol in alphabet.return_alphabet:
+            if not stack:
+                raise ValueError("return with empty stack")
+            call_symbol = stack.pop()
+            encoded.append((symbol, call_symbol))
+        else:
+            encoded.append(symbol)
+    return tuple(encoded)
+
+
+def papni_encode_samples(
+    samples: Sequence[Sequence[Any]],
+    alphabet: DyckAlphabet,
+    *,
+    drop_non_well_matched: bool = True,
+) -> list[tuple[Any, ...]]:
+    """Filter and encode samples for RPNI over the stack-aware alphabet."""
+    encoded: list[tuple[Any, ...]] = []
+    for word in samples:
+        seq = tuple(word)
+        if not is_well_matched(seq, alphabet):
+            if drop_non_well_matched:
+                continue
+            raise ValueError(f"sample {seq!r} is not well-matched")
+        encoded.append(papni_encode(seq, alphabet))
+    return encoded
+
+
+def _decode_symbol(symbol: Any, alphabet: DyckAlphabet) -> tuple[str, Any, Any | None]:
+    if symbol in alphabet.call_alphabet:
+        return KIND_CALL, symbol, None
+    if symbol in alphabet.internal_alphabet:
+        return KIND_INTERNAL, symbol, None
+    if isinstance(symbol, tuple) and len(symbol) == 2:
+        return_symbol, call_symbol = symbol
+        if return_symbol in alphabet.return_alphabet and call_symbol in alphabet.call_alphabet:
+            return KIND_RETURN, return_symbol, call_symbol
+    raise ValueError(f"unknown stack-aware symbol {symbol!r}")
+
+
+def sofic_dyck_shift_from_papni_dfa(
+    dfa: DFA,
+    alphabet: DyckAlphabet,
+    *,
+    positive_traces: Sequence[Sequence[Any]] | None = None,
+) -> SoficDyckShift:
+    """Convert a PAPNI-learned DFA over stack-aware symbols into a ``SoficDyckShift``."""
+    if not dfa.initial_states:
+        raise ValueError("DFA requires an initial state")
+    initial = next(iter(dfa.initial_states))
+    reachable = dfa.graph.forward_reachable({initial})
+
+    shift = SoficDyckShift(
+        call_alphabet=alphabet.call_alphabet,
+        return_alphabet=alphabet.return_alphabet,
+        internal_alphabet=alphabet.internal_alphabet,
+    )
+
+    for state in sorted(reachable, key=repr):
+        shift.graph.add_state(state)
+
+    call_by_symbol: dict[Any, list[TransitionRef]] = {symbol: [] for symbol in alphabet.call_alphabet}
+    return_by_pair: dict[tuple[Any, Any], list[TransitionRef]] = {}
+
+    for source in reachable:
+        for transition in dfa.graph.out_transitions(source):
+            symbol = transition.data.get(ATTR_SYMBOL)
+            if symbol is None:
+                continue
+            kind, visible, matched_call = _decode_symbol(symbol, alphabet)
+            target = transition.target
+            if kind == KIND_CALL:
+                ref = shift.add_call_transition(source, target, visible)
+                call_by_symbol[visible].append(ref)
+            elif kind == KIND_INTERNAL:
+                shift.add_internal_transition(source, target, visible)
+            else:
+                assert matched_call is not None
+                ref = shift.add_return_transition(source, target, visible)
+                return_by_pair.setdefault((visible, matched_call), []).append(ref)
+
+    if positive_traces:
+        _infer_matched_edges_from_traces(shift, dfa, alphabet, positive_traces)
+    else:
+        for (_return_symbol, call_symbol), return_refs in return_by_pair.items():
+            for return_ref in return_refs:
+                for call_ref in call_by_symbol.get(call_symbol, ()):
+                    shift.add_matched_pair(call_ref, return_ref)
+
+    shift = _trim_shift_to_reachable(shift, initial)
+    shift.validate()
+    return shift
+
+
+def _trim_shift_to_reachable(shift: SoficDyckShift, initial: Hashable) -> SoficDyckShift:
+    """Drop unreachable states and zero-outdegree control states."""
+    if not shift.graph.has_state(initial):
+        return shift
+    active = set(shift.graph.forward_reachable({initial}))
+
+    dead = {state for state in active if not any(True for _ in shift.graph.out_transitions(state))}
+    active -= dead
+
+    trimmed = SoficDyckShift(
+        call_alphabet=shift.call_alphabet,
+        return_alphabet=shift.return_alphabet,
+        internal_alphabet=shift.internal_alphabet,
+        matched_edges=shift.matched_edges,
+        symbol_alphabet=shift.symbol_alphabet,
+    )
+    for state in active:
+        trimmed.graph.add_state(state, **shift.graph.state_attrs(state))
+
+    edge_map: dict[TransitionRef, TransitionRef] = {}
+    for transition in shift.transitions():
+        if transition.source not in active:
+            continue
+        target = transition.target if transition.target in active else transition.source
+        data = dict(transition.data)
+        key = trimmed.graph.add_transition(transition.source, target, **data)
+        edge_map[transition_ref(transition)] = (transition.source, target, key)
+
+    trimmed.matched_edges = frozenset(
+        (edge_map[call_ref], edge_map[return_ref])
+        for call_ref, return_ref in shift.matched_edges
+        if call_ref in edge_map and return_ref in edge_map
+    )
+    return trimmed
+
+
+def _infer_matched_edges_from_traces(
+    shift: SoficDyckShift,
+    dfa: DFA,
+    alphabet: DyckAlphabet,
+    traces: Sequence[Sequence[Any]],
+) -> None:
+    """Record matched call-return pairs observed when replaying positive traces."""
+    if not dfa.initial_states:
+        return
+    initial = next(iter(dfa.initial_states))
+
+    for word in traces:
+        if not is_well_matched(word, alphabet):
+            continue
+        encoded = papni_encode(word, alphabet)
+        dfa_state = initial
+        config_state = initial
+        stack: list[TransitionRef] = []
+
+        for symbol in encoded:
+            kind, visible, matched_call = _decode_symbol(symbol, alphabet)
+            successors = list(dfa.graph.out_transitions(dfa_state))
+            dfa_transition = next(
+                (transition for transition in successors if transition.data.get(ATTR_SYMBOL) == symbol),
+                None,
+            )
+            if dfa_transition is None:
+                break
+            dfa_state = dfa_transition.target
+
+            if kind == KIND_CALL:
+                ref = _select_transition(shift, config_state, dfa_state, KIND_CALL, visible)
+                if ref is None:
+                    break
+                stack.append(ref)
+                config_state = dfa_state
+            elif kind == KIND_INTERNAL:
+                ref = _select_transition(shift, config_state, dfa_state, KIND_INTERNAL, visible)
+                if ref is None:
+                    break
+                config_state = dfa_state
+            else:
+                assert matched_call is not None
+                ref = _select_transition(shift, config_state, dfa_state, KIND_RETURN, visible)
+                if ref is None or not stack:
+                    break
+                call_ref = stack.pop()
+                shift.add_matched_pair(call_ref, ref)
+                config_state = dfa_state
+
+
+def _select_transition(
+    shift: SoficDyckShift,
+    source: Hashable,
+    target: Hashable,
+    kind: str,
+    symbol: Any,
+) -> TransitionRef | None:
+    for transition in shift.graph.out_transitions(source):
+        if transition.target != target:
+            continue
+        if transition.data.get(ATTR_KIND) != kind:
+            continue
+        if transition.data.get(ATTR_SYMBOL) != symbol:
+            continue
+        return transition_ref(transition)
+    return None
+
+
+def learn_sofic_dyck_shift_papni(
+    positive: Sequence[Sequence[Any]],
+    negative: Sequence[Sequence[Any]] | None = None,
+    *,
+    alphabet: DyckAlphabet,
+) -> SoficDyckShift:
+    """Learn a ``SoficDyckShift`` topology from labeled samples via PAPNI + RPNI."""
+    encoded_positive = papni_encode_samples(positive, alphabet)
+    if not encoded_positive:
+        raise ValueError("no well-matched positive samples remain after PAPNI filtering")
+
+    encoded_negative: list[tuple[Any, ...]] = []
+    if negative:
+        for word in negative:
+            seq = tuple(word)
+            if not is_well_matched(seq, alphabet):
+                continue
+            encoded_negative.append(papni_encode(seq, alphabet))
+
+    dfa = learn_dfa_rpni(encoded_positive, encoded_negative)
+    well_matched_positive = [tuple(word) for word in positive if is_well_matched(word, alphabet)]
+    return sofic_dyck_shift_from_papni_dfa(
+        dfa,
+        alphabet,
+        positive_traces=well_matched_positive,
+    )

sofic/automata/regex.py

@@ -0,0 +1,128 @@
+"""Automata to regular-expression conversion."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Any
+
+from sofic.automata.base import LabeledAutomaton
+from sofic.graph import ATTR_SYMBOL, EPSILON
+
+
+@dataclass(frozen=True, slots=True)
+class _Regex:
+    kind: str
+    parts: tuple[Any, ...] = ()
+
+    def pattern(self) -> str:
+        if self.kind == "empty":
+            return "(?!)"
+        if self.kind == "epsilon":
+            return "(?:)"
+        if self.kind == "literal":
+            return re.escape(str(self.parts[0]))
+        if self.kind == "union":
+            return "(?:" + "|".join(part.pattern() for part in self.parts) + ")"
+        if self.kind == "concat":
+            return "".join(_atom(part) for part in self.parts)
+        if self.kind == "star":
+            return _atom(self.parts[0]) + "*"
+        raise ValueError(f"unknown regex node {self.kind!r}")
+
+
+_EMPTY = _Regex("empty")
+_EPSILON = _Regex("epsilon")
+
+
+def automaton_to_regex(automaton: LabeledAutomaton) -> str:
+    """Return a Python-regex-compatible expression for ``automaton``.
+
+    The result uses ``(?!)`` for the empty language and ``(?:)`` for epsilon.
+    Symbols are rendered with ``str(symbol)`` and escaped for Python's
+    :mod:`re` engine.
+    """
+    start = object()
+    final = object()
+    states = sorted(automaton.states(), key=repr)
+    remaining: list[Any] = [start, *states, final]
+    labels: dict[tuple[Any, Any], _Regex] = {}
+
+    for state in automaton.initial_states:
+        labels[(start, state)] = _union(labels.get((start, state), _EMPTY), _EPSILON)
+    for state in automaton.accepting_states:
+        labels[(state, final)] = _union(labels.get((state, final), _EMPTY), _EPSILON)
+    for transition in automaton.transitions():
+        symbol = transition.data.get(ATTR_SYMBOL)
+        if symbol is None:
+            continue
+        label = _EPSILON if symbol is EPSILON else _Regex("literal", (symbol,))
+        key = (transition.source, transition.target)
+        labels[key] = _union(labels.get(key, _EMPTY), label)
+
+    for state in states:
+        others = [candidate for candidate in remaining if candidate != state]
+        loop = _star(labels.get((state, state), _EMPTY))
+        for source in others:
+            left = labels.get((source, state), _EMPTY)
+            if left == _EMPTY:
+                continue
+            for target in others:
+                right = labels.get((state, target), _EMPTY)
+                if right == _EMPTY:
+                    continue
+                key = (source, target)
+                labels[key] = _union(labels.get(key, _EMPTY), _concat(left, loop, right))
+        labels = {key: value for key, value in labels.items() if state not in key}
+        remaining = others
+
+    return labels.get((start, final), _EMPTY).pattern()
+
+
+def _union(*terms: _Regex) -> _Regex:
+    parts: list[_Regex] = []
+    for term in terms:
+        if term == _EMPTY:
+            continue
+        if term.kind == "union":
+            parts.extend(term.parts)
+        else:
+            parts.append(term)
+    unique = sorted(set(parts), key=lambda part: part.pattern())
+    if not unique:
+        return _EMPTY
+    if len(unique) == 1:
+        return unique[0]
+    return _Regex("union", tuple(unique))
+
+
+def _concat(*terms: _Regex) -> _Regex:
+    parts: list[_Regex] = []
+    for term in terms:
+        if term == _EMPTY:
+            return _EMPTY
+        if term == _EPSILON:
+            continue
+        if term.kind == "concat":
+            parts.extend(term.parts)
+        else:
+            parts.append(term)
+    if not parts:
+        return _EPSILON
+    if len(parts) == 1:
+        return parts[0]
+    return _Regex("concat", tuple(parts))
+
+
+def _star(term: _Regex) -> _Regex:
+    if term in {_EMPTY, _EPSILON}:
+        return _EPSILON
+    if term.kind == "star":
+        return term
+    return _Regex("star", (term,))
+
+
+def _atom(term: _Regex) -> str:
+    if term.kind in {"literal", "epsilon", "empty", "star"}:
+        return term.pattern()
+    return "(?:" + term.pattern() + ")"

sofic/automata/rfsa.py

@@ -0,0 +1,35 @@
+"""Residual and canonical RFSA automata."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from sofic.automata.languages.base import RegularLanguage
+from sofic.automata.nfa import NFA
+
+if TYPE_CHECKING:
+    from sofic.automata.observation import ObservationTable
+
+
+class ResidualFiniteStateAutomaton(NFA):
+    """NFA whose states accept residual languages of the recognized language."""
+
+    def validate(self) -> None:
+        super().validate()
+        # Phase 2: verify each state's right language is in Res(L(R))
+
+
+class CanonicalRFSA(ResidualFiniteStateAutomaton):
+    """Canonical residual finite-state automaton R(L)."""
+
+    @classmethod
+    def from_language(cls, language: RegularLanguage | NFA, **kwargs: Any) -> CanonicalRFSA:
+        from sofic.automata.canonical_extraction import canonical_rfsa_from_language
+
+        return canonical_rfsa_from_language(language)
+
+    @classmethod
+    def from_observation_table(cls, table: ObservationTable, **kwargs: Any) -> CanonicalRFSA:
+        from sofic.automata.canonical_extraction import observation_to_canonical_rfsa
+
+        return observation_to_canonical_rfsa(table)

sofic/automata/rpni.py

@@ -0,0 +1,193 @@
+"""Passive DFA learning via the RPNI state-merging algorithm."""
+
+from __future__ import annotations
+
+from collections.abc import Hashable, Sequence
+from dataclasses import dataclass, field
+from typing import Any
+
+from sofic.automata.dfa import DFA
+
+__all__ = ["learn_dfa_rpni"]
+
+
+@dataclass
+class _PTANode:
+    """Prefix-tree acceptor node."""
+
+    transitions: dict[Any, int] = field(default_factory=dict)
+    accepting: bool = False
+
+
+def _build_pta(
+    positive: Sequence[Sequence[Any]],
+    negative: Sequence[Sequence[Any]],
+) -> tuple[list[_PTANode], dict[tuple[Any, ...], int]]:
+    """Build a prefix-tree acceptor from labeled samples."""
+    nodes: list[_PTANode] = [_PTANode()]
+    index: dict[tuple[Any, ...], int] = {(): 0}
+
+    def ensure(prefix: tuple[Any, ...]) -> int:
+        if prefix not in index:
+            index[prefix] = len(nodes)
+            nodes.append(_PTANode())
+        return index[prefix]
+
+    for word in positive:
+        prefix: tuple[Any, ...] = ()
+        for symbol in word:
+            node_id = index[prefix]
+            next_prefix = prefix + (symbol,)
+            if symbol not in nodes[node_id].transitions:
+                nodes[node_id].transitions[symbol] = ensure(next_prefix)
+            prefix = next_prefix
+        nodes[index[prefix]].accepting = True
+
+    for word in negative:
+        prefix = ()
+        for symbol in word:
+            node_id = index[prefix]
+            next_prefix = prefix + (symbol,)
+            if symbol not in nodes[node_id].transitions:
+                nodes[node_id].transitions[symbol] = ensure(next_prefix)
+            prefix = next_prefix
+        nodes[index[prefix]].accepting = False
+
+    return nodes, index
+
+
+def _collect_alphabet(
+    positive: Sequence[Sequence[Any]],
+    negative: Sequence[Sequence[Any]],
+) -> tuple[Any, ...]:
+    symbols: set[Any] = set()
+    for word in positive:
+        symbols.update(word)
+    for word in negative:
+        symbols.update(word)
+    return tuple(sorted(symbols, key=repr))
+
+
+def _compatible_merge(
+    nodes: list[_PTANode],
+    left: int,
+    right: int,
+    merge_map: dict[int, int],
+) -> bool:
+    """Return whether merging ``right`` into ``left`` is RPNI-compatible."""
+    if nodes[left].accepting != nodes[right].accepting:
+        return False
+    symbols = set(nodes[left].transitions) | set(nodes[right].transitions)
+    for symbol in symbols:
+        left_target = nodes[left].transitions.get(symbol)
+        right_target = nodes[right].transitions.get(symbol)
+        if left_target is None or right_target is None:
+            if left_target is not None or right_target is not None:
+                return False
+            continue
+        if merge_map.get(left_target, left_target) != merge_map.get(right_target, right_target):
+            return False
+    return True
+
+
+def _apply_merge(nodes: list[_PTANode], left: int, right: int, merge_map: dict[int, int]) -> None:
+    """Merge PTA node ``right`` into ``left``."""
+    merge_map[right] = left
+    for symbol, target in nodes[right].transitions.items():
+        if symbol not in nodes[left].transitions:
+            nodes[left].transitions[symbol] = target
+    nodes[right].transitions.clear()
+    nodes[right].accepting = nodes[left].accepting
+
+
+def _rpni_merge(nodes: list[_PTANode]) -> dict[int, int]:
+    """Greedy RPNI state merging on a PTA."""
+    merge_map: dict[int, int] = {index: index for index in range(len(nodes))}
+
+    def representative(node_id: int) -> int:
+        while merge_map[node_id] != node_id:
+            merge_map[node_id] = merge_map[merge_map[node_id]]
+            node_id = merge_map[node_id]
+        return node_id
+
+    changed = True
+    while changed:
+        changed = False
+        reps = sorted({representative(index) for index in range(len(nodes))})
+        for left in reps:
+            for right in reps:
+                if left >= right:
+                    continue
+                if not _compatible_merge(nodes, left, right, merge_map):
+                    continue
+                _apply_merge(nodes, left, right, merge_map)
+                changed = True
+                break
+            if changed:
+                break
+    return merge_map
+
+
+def _pta_to_dfa(
+    nodes: list[_PTANode],
+    merge_map: dict[int, int],
+    alphabet: Sequence[Any],
+) -> DFA:
+    def representative(node_id: int) -> int:
+        while merge_map[node_id] != node_id:
+            merge_map[node_id] = merge_map[merge_map[node_id]]
+            node_id = merge_map[node_id]
+        return node_id
+
+    blocks: dict[int, list[int]] = {}
+    for node_id in range(len(nodes)):
+        rep = representative(node_id)
+        blocks.setdefault(rep, []).append(node_id)
+
+    rep_to_label: dict[int, Hashable] = {}
+    for block_id, (rep, _members) in enumerate(sorted(blocks.items(), key=lambda item: min(item[1]))):
+        rep_to_label[rep] = f"q{block_id}"
+
+    dfa = DFA(input_alphabet=frozenset(alphabet))
+    for label in rep_to_label.values():
+        dfa.graph.add_state(label)
+
+    initial = rep_to_label[representative(0)]
+    dfa.initial_states = frozenset({initial})
+
+    accepting: set[Hashable] = set()
+    for rep, members in blocks.items():
+        label = rep_to_label[rep]
+        if any(nodes[member].accepting for member in members):
+            accepting.add(label)
+        rep_node = nodes[rep]
+        for symbol in alphabet:
+            target = rep_node.transitions.get(symbol)
+            if target is None:
+                continue
+            target_label = rep_to_label[representative(target)]
+            dfa.add_transition(label, target_label, symbol)
+
+    dfa.accepting_states = frozenset(accepting)
+    dfa.validate()
+    return dfa
+
+
+def learn_dfa_rpni(
+    positive: Sequence[Sequence[Any]],
+    negative: Sequence[Sequence[Any]] | None = None,
+) -> DFA:
+    """Learn a minimal compatible DFA from positive and negative samples.
+
+    Implements the classic RPNI greedy merge strategy on a prefix-tree
+    acceptor built from the sample sets.
+    """
+    pos = [tuple(word) for word in positive]
+    neg = [tuple(word) for word in (negative or ())]
+    if not pos and not neg:
+        raise ValueError("at least one positive or negative sample is required")
+
+    alphabet = _collect_alphabet(pos, neg)
+    nodes, _index = _build_pta(pos, neg)
+    merge_map = _rpni_merge(nodes)
+    return _pta_to_dfa(nodes, merge_map, alphabet)