c2score 0.1.0__py3-none-any.whl

c2score/__init__.py

@@ -0,0 +1,25 @@
+"""C2-score: quantitative quality evaluation of property graph schemas."""
+
+from importlib.metadata import version
+
+from .errors import C2ScoreError, InstanceLoadError, ParameterError, SchemaError
+from .evaluate import EvaluationResult, evaluate
+from .loaders import load_neo4j
+from .model import Instance, Schema
+from .schema_json import load_schema_json
+
+__version__ = version("c2score")
+
+__all__ = [
+    "load_neo4j",
+    "load_schema_json",
+    "evaluate",
+    "Instance",
+    "Schema",
+    "EvaluationResult",
+    "C2ScoreError",
+    "SchemaError",
+    "InstanceLoadError",
+    "ParameterError",
+    "__version__",
+]

c2score/_instance_json.py

@@ -0,0 +1,165 @@
+"""Saving and restoring Abs(G) as JSON (``Instance.to_json`` / ``Instance.from_json``).
+
+The file holds the summary, not the graph::
+
+    {
+      "format": "c2score-instance",
+      "version": 1,
+      "node_types": [{"labels": [...], "properties": {key: {"type": ..., "required": ...}}}],
+      "edge_types": [{"labels": [...], "source_labels": [...], "target_labels": [...],
+                      "cardinality": {"out": "0..1", "in": "1..*"}, "properties": {...}}]
+    }
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from collections.abc import Mapping
+from typing import Any, Union
+
+from .cardinality import Cardinality, format_interval, parse_interval
+from .datatypes import DataType
+from .errors import InstanceLoadError
+from .model import Instance, InstanceEdgeType, InstanceNodeType, PropertySpec
+
+FORMAT = "c2score-instance"
+VERSION = 1
+
+_TOP_LEVEL_KEYS = ("format", "version", "node_types", "edge_types")
+_NODE_KEYS = ("labels", "properties")
+_EDGE_KEYS = ("labels", "source_labels", "target_labels", "cardinality", "properties")
+
+PathLike = Union[str, "os.PathLike[str]"]
+
+
+def _dump_properties(properties: Mapping[str, PropertySpec]) -> dict[str, Any]:
+    return {
+        key: {"type": spec.type.value, "required": spec.required}
+        for key, spec in sorted(properties.items())
+    }
+
+
+def instance_to_dict(instance: Instance) -> dict[str, Any]:
+    """Return the JSON-serializable form of ``instance``."""
+    return {
+        "format": FORMAT,
+        "version": VERSION,
+        "node_types": [
+            {"labels": sorted(node.labels), "properties": _dump_properties(node.properties)}
+            for node in instance.node_types
+        ],
+        "edge_types": [
+            {
+                "labels": sorted(edge.labels),
+                "source_labels": sorted(edge.source_labels),
+                "target_labels": sorted(edge.target_labels),
+                "cardinality": {
+                    "out": format_interval(edge.cardinality.out),
+                    "in": format_interval(edge.cardinality.in_),
+                },
+                "properties": _dump_properties(edge.properties),
+            }
+            for edge in instance.edge_types
+        ],
+    }
+
+
+def save(instance: Instance, path: PathLike) -> None:
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(instance_to_dict(instance), f, ensure_ascii=False, indent=2)
+        f.write("\n")
+
+
+def _check_keys(value: Mapping[str, Any], keys: tuple[str, ...], where: str) -> None:
+    """Reject missing and unknown keys, so that a misspelled key is not silently ignored."""
+    for key in keys:
+        if key not in value:
+            raise ValueError(f"{where}missing key {key!r}")
+    unknown = [key for key in value if key not in keys]
+    if unknown:
+        listed = ", ".join(repr(key) for key in unknown)
+        raise ValueError(f"{where}unknown key{'s' if len(unknown) > 1 else ''} {listed}")
+
+
+def _labels(value: Any, where: str) -> frozenset[str]:
+    if not isinstance(value, list) or not all(isinstance(label, str) for label in value):
+        raise ValueError(f"{where}: expected a list of strings, got {value!r}")
+    return frozenset(value)
+
+
+def _load_properties(value: Any, where: str) -> dict[str, PropertySpec]:
+    if not isinstance(value, dict):
+        raise ValueError(f"{where}: expected an object, got {value!r}")
+    properties = {}
+    for key, spec in value.items():
+        if not isinstance(spec, dict) or set(spec) != {"type", "required"}:
+            raise ValueError(f"{where}.{key}: expected {{\"type\", \"required\"}}, got {spec!r}")
+        if not isinstance(spec["required"], bool):
+            raise ValueError(f"{where}.{key}.required: expected true or false, got {spec['required']!r}")
+        try:
+            data_type = DataType(spec["type"])
+        except ValueError:
+            raise ValueError(f"{where}.{key}.type: unknown type {spec['type']!r}") from None
+        properties[key] = PropertySpec(data_type, spec["required"])
+    return properties
+
+
+def _list(data: Mapping[str, Any], key: str) -> list[Any]:
+    value = data.get(key)
+    if not isinstance(value, list) or not all(isinstance(item, dict) for item in value):
+        raise ValueError(f"{key}: expected a list of objects")
+    return value
+
+
+def instance_from_dict(data: Any) -> Instance:
+    """Rebuild an :class:`Instance` from :func:`instance_to_dict` output.
+
+    Raises ``ValueError`` on malformed input, including missing and unknown keys.
+    """
+    if not isinstance(data, dict) or data.get("format") != FORMAT:
+        raise ValueError(f"not a c2score instance file (expected \"format\": {FORMAT!r})")
+    version = data.get("version")
+    # The version is an integer; True and 1.0 compare equal to 1 but are not accepted.
+    if type(version) is not int or version != VERSION:
+        raise ValueError(f"unsupported version {version!r}; this c2score reads version {VERSION}")
+    _check_keys(data, _TOP_LEVEL_KEYS, "")
+    node_types = []
+    for i, node in enumerate(_list(data, "node_types")):
+        where = f"node_types[{i}]"
+        _check_keys(node, _NODE_KEYS, f"{where}: ")
+        node_types.append(
+            InstanceNodeType(
+                _labels(node["labels"], f"{where}.labels"),
+                _load_properties(node["properties"], f"{where}.properties"),
+            )
+        )
+    edge_types = []
+    for i, edge in enumerate(_list(data, "edge_types")):
+        where = f"edge_types[{i}]"
+        _check_keys(edge, _EDGE_KEYS, f"{where}: ")
+        cardinality = edge["cardinality"]
+        if not isinstance(cardinality, dict) or set(cardinality) != {"out", "in"}:
+            raise ValueError(f"{where}.cardinality: expected {{\"out\", \"in\"}}, got {cardinality!r}")
+        edge_types.append(
+            InstanceEdgeType(
+                _labels(edge["labels"], f"{where}.labels"),
+                _labels(edge["source_labels"], f"{where}.source_labels"),
+                _labels(edge["target_labels"], f"{where}.target_labels"),
+                Cardinality(parse_interval(cardinality["out"]), parse_interval(cardinality["in"])),
+                _load_properties(edge["properties"], f"{where}.properties"),
+            )
+        )
+    return Instance(tuple(node_types), tuple(edge_types))
+
+
+def load(path: PathLike) -> Instance:
+    origin = os.fspath(path)
+    with open(path, "rb") as f:
+        raw = f.read()
+    try:
+        return instance_from_dict(json.loads(raw.decode("utf-8")))
+    except (UnicodeDecodeError, json.JSONDecodeError) as e:
+        raise InstanceLoadError(f"invalid instance file ({origin}): not valid UTF-8 JSON: {e}") from e
+    except ValueError as e:
+        raise InstanceLoadError(f"invalid instance file ({origin}): {e}") from e

c2score/cardinality.py

@@ -0,0 +1,94 @@
+"""Cardinality intervals and their similarity (D_card).
+
+An interval is a pair ``(lower, upper)`` with ``lower`` in ``{0, 1}`` and
+``upper`` in ``{1, "*"}``. Only the four intervals ``0..1``, ``0..*``,
+``1..1`` and ``1..*`` exist; observed degree bounds are always rounded to one
+of them.
+"""
+
+from __future__ import annotations
+
+from typing import NamedTuple, Union
+
+STAR = "*"
+
+Bound = Union[int, str]
+Interval = tuple[int, Bound]
+
+ZERO_ONE: Interval = (0, 1)
+ZERO_MANY: Interval = (0, STAR)
+ONE_ONE: Interval = (1, 1)
+ONE_MANY: Interval = (1, STAR)
+
+_BY_NAME: dict[str, Interval] = {
+    "0..1": ZERO_ONE,
+    "0..*": ZERO_MANY,
+    "1..1": ONE_ONE,
+    "1..*": ONE_MANY,
+}
+_NAME_OF: dict[Interval, str] = {interval: name for name, interval in _BY_NAME.items()}
+
+
+class Cardinality(NamedTuple):
+    """The out-degree and in-degree intervals of an edge type."""
+
+    out: Interval
+    in_: Interval
+
+    def __str__(self) -> str:
+        return f"out {format_interval(self.out)} / in {format_interval(self.in_)}"
+
+
+#: The cardinality assumed for an edge type that has none.
+DEFAULT_CARDINALITY = Cardinality(ZERO_MANY, ZERO_MANY)
+
+
+def parse_interval(text: str) -> Interval:
+    """Parse one of ``"0..1"``, ``"0..*"``, ``"1..1"``, ``"1..*"``."""
+    try:
+        return _BY_NAME[text]
+    except (KeyError, TypeError):
+        allowed = ", ".join(repr(name) for name in _BY_NAME)
+        raise ValueError(f"invalid interval {text!r}; expected one of {allowed}") from None
+
+
+def format_interval(interval: Interval) -> str:
+    """Return the text form of one of the four intervals."""
+    try:
+        return _NAME_OF[tuple(interval)]
+    except (KeyError, TypeError):
+        raise ValueError(f"not one of the four intervals: {interval!r}") from None
+
+
+def round_observed(min_degree: int, max_degree: int) -> Interval:
+    """Round observed degree bounds ``[min, max]`` to one of the four intervals.
+
+    The lower bound becomes 0 if ``min`` is 0 and 1 otherwise; the upper bound
+    becomes 1 if ``max`` is at most 1 and ``"*"`` otherwise.
+    """
+    if min_degree < 0 or max_degree < min_degree or max_degree < 1:
+        raise ValueError(f"invalid observed degree bounds [{min_degree}, {max_degree}]")
+    lower = 0 if min_degree == 0 else 1
+    upper: Bound = 1 if max_degree <= 1 else STAR
+    return (lower, upper)
+
+
+def bd(x: Bound, y: Bound) -> float:
+    """Similarity of two bounds (Eq. 33)."""
+    x_star = x == STAR
+    y_star = y == STAR
+    if x_star and y_star:
+        return 1.0
+    if x_star or y_star:
+        return 0.0
+    return (min(x, y) + 1) / (max(x, y) + 1)
+
+
+def d_int(a: Interval, b: Interval) -> float:
+    """Similarity of two intervals (Eq. 32)."""
+    return (bd(a[0], b[0]) + bd(a[1], b[1])) / 2
+
+
+def d_card(c: Cardinality, c2: Cardinality) -> float:
+    """Similarity of two cardinalities (Eq. 31)."""
+    return (d_int(c.out, c2.out) + d_int(c.in_, c2.in_)) / 2

c2score/conciseness.py

@@ -0,0 +1,191 @@
+"""Conciseness: which written definitions are redundant (Section 4.7).
+
+Each definition is removed in turn, together with everything that depends on
+it, and the coverage of the reduced schema is compared with the original.
+
+When no other definition inherits from the removed one (and, for a node
+definition, no edge definition that goes with it), the remaining definitions
+flatten exactly as before: S+ of the reduced schema is S+ minus the elements
+that the removal takes away. The coverage is then read off the original
+similarities. Otherwise the reduced schema is flattened again.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Collection
+from dataclasses import dataclass, replace
+from typing import Optional, TypeVar
+
+from .coverage import CoverageTables
+from .flatten import FlatSchema, flatten, inherited_endpoint
+from .model import EdgeTypeDef, NodeTypeDef, Schema
+from .similarity import Similarity
+
+_D = TypeVar("_D", NodeTypeDef, EdgeTypeDef)
+
+
+@dataclass(frozen=True)
+class Removal:
+    """The effect of removing one definition."""
+
+    type_id: str
+    delta_v: float
+    delta_e: float
+    redundant: bool
+
+
+@dataclass(frozen=True)
+class ConcisenessResult:
+    con_v: float
+    con_e: float
+    theta_v: float
+    theta_e: float
+    node_removals: tuple[Removal, ...]
+    edge_removals: tuple[Removal, ...]
+    #: The definitions whose removal needed a new flattening (the others were incremental).
+    reflattened: tuple[str, ...] = ()
+
+
+def reduce_schema(schema: Schema, type_id: str) -> Schema:
+    """S - d: remove a definition and, repeatedly, what no longer holds without it.
+
+    Removed definitions are dropped from the remaining ``supertypes``. An edge
+    definition whose endpoint (possibly inherited) is removed or no longer
+    determined is removed as well.
+    """
+    removed = {type_id}
+
+    def kept(definition: _D) -> _D:
+        if not any(parent in removed for parent in definition.supertypes):
+            return definition
+        return replace(definition, supertypes=tuple(p for p in definition.supertypes if p not in removed))
+
+    while True:
+        reduced = Schema(
+            node_types=tuple(kept(n) for n in schema.node_types if n.id not in removed),
+            edge_types=tuple(kept(e) for e in schema.edge_types if e.id not in removed),
+        )
+        broken = set()
+        for edge in reduced.edge_types:
+            for end in ("source", "target"):
+                endpoint = inherited_endpoint(reduced, edge, end)
+                if endpoint is None or endpoint in removed:
+                    broken.add(edge.id)
+        if not broken:
+            return reduced
+        removed |= broken
+
+
+def _ratio(redundant: int, total: int) -> float:
+    """1 - redundant / total, or 1 if there is nothing to remove."""
+    return 1.0 if total == 0 else 1 - redundant / total
+
+
+class _Shortcuts:
+    """What a removal takes away from S+, for the removals that leave the rest of S+ unchanged."""
+
+    def __init__(self, schema: Schema, flat: FlatSchema) -> None:
+        children: dict[str, list[str]] = {}
+        for definition in (*schema.node_types, *schema.edge_types):
+            for parent in definition.supertypes:
+                children.setdefault(parent, []).append(definition.id)
+        attached: dict[str, set[str]] = {}
+        for edge in schema.edge_types:
+            for end in ("source", "target"):
+                endpoint = inherited_endpoint(schema, edge, end)
+                if endpoint is not None:
+                    attached.setdefault(endpoint, set()).add(edge.id)
+        self._children = children
+        self._attached = attached
+        self._node_column = {s.id: j for j, s in enumerate(flat.node_types)}
+        self._touching: dict[str, set[int]] = {}
+        self._only_from: dict[str, set[int]] = {}
+        for j, e in enumerate(flat.edge_types):
+            self._touching.setdefault(e.source.id, set()).add(j)
+            self._touching.setdefault(e.target.id, set()).add(j)
+            if len(e.origins) == 1:
+                self._only_from.setdefault(next(iter(e.origins)), set()).add(j)
+
+    def node(self, type_id: str) -> Optional[tuple[int, Collection[int]]]:
+        """For a node definition: its column in V(S+) and the columns of E(S+) that go with it.
+
+        ``None`` if the removal changes other elements of S+. That happens if
+        some definition inherits from it, or from an edge definition removed
+        with it (one whose endpoint, possibly inherited, is this node).
+        Otherwise the removed edge definitions only produce elements whose
+        source or target is this node (it has no descendants), and every
+        other element survives with the same content.
+        """
+        if self._children.get(type_id):
+            return None
+        removed_edges = self._attached.get(type_id, set())
+        for edge_id in removed_edges:
+            if any(child not in removed_edges for child in self._children.get(edge_id, ())):
+                return None
+        return self._node_column[type_id], self._touching.get(type_id, set())
+
+    def edge(self, type_id: str) -> Optional[Collection[int]]:
+        """For an edge definition: the columns of E(S+) produced by it alone.
+
+        ``None`` if some definition inherits from it.
+        """
+        if self._children.get(type_id):
+            return None
+        return self._only_from.get(type_id, set())
+
+
+def conciseness(
+    schema: Schema,
+    flat: FlatSchema,
+    base_cov_v: float,
+    base_cov_e: float,
+    similarity: Similarity,
+    redundancy_ratio: float,
+    *,
+    tables: Optional[CoverageTables] = None,
+) -> ConcisenessResult:
+    """Con_V and Con_E. ``flat``, the base coverages and ``tables`` belong to the original schema."""
+    n_v, n_e = len(flat.node_types), len(flat.edge_types)
+    theta_v = redundancy_ratio * base_cov_v / n_v if n_v else 0.0
+    theta_e = redundancy_ratio * base_cov_e / n_e if n_e else 0.0
+    if tables is None:
+        tables = CoverageTables(flat, similarity)
+    shortcuts = _Shortcuts(schema, flat)
+    reflattened: list[str] = []
+
+    def judge(type_id: str, cov_v_after: float, cov_e_after: float) -> Removal:
+        delta_v = base_cov_v - cov_v_after
+        delta_e = base_cov_e - cov_e_after
+        redundant = (delta_v < theta_v or delta_v <= 0) and (delta_e < theta_e or delta_e <= 0)
+        return Removal(type_id, delta_v, delta_e, redundant)
+
+    def again(type_id: str) -> Removal:
+        reflattened.append(type_id)
+        reduced = CoverageTables(flatten(reduce_schema(schema, type_id)), similarity)
+        return judge(type_id, reduced.cov_v_value, reduced.cov_e_value)
+
+    def node_removal(type_id: str) -> Removal:
+        taken = shortcuts.node(type_id)
+        if taken is None:
+            return again(type_id)
+        column, edges = taken
+        return judge(type_id, tables.cov_v_without(column), tables.cov_e_without(edges))
+
+    def edge_removal(type_id: str) -> Removal:
+        edges = shortcuts.edge(type_id)
+        if edges is None:
+            return again(type_id)
+        # The node types do not change, so Cov_V is computed from the same values.
+        return judge(type_id, tables.cov_v_value, tables.cov_e_without(edges))
+
+    node_removals = tuple(node_removal(n.id) for n in schema.node_types)
+    edge_removals = tuple(edge_removal(e.id) for e in schema.edge_types)
+    return ConcisenessResult(
+        con_v=_ratio(sum(r.redundant for r in node_removals), len(node_removals)),
+        con_e=_ratio(sum(r.redundant for r in edge_removals), len(edge_removals)),
+        theta_v=theta_v,
+        theta_e=theta_e,
+        node_removals=node_removals,
+        edge_removals=edge_removals,
+        reflattened=tuple(reflattened),
+    )