celatim 0.2.2__tar.gz → 0.2.3__tar.gz

{celatim-0.2.2 → celatim-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: celatim
-Version: 0.2.2
+Version: 0.2.3
 Summary: Typed secure file transfer and reproducible covert-channel measurement for IETF protocol fields.
 Keywords: covert-channels,network-measurement,network-protocols,privacy,steganography
 Author: Michael Bommarito

{celatim-0.2.2 → celatim-0.2.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "celatim"
-version = "0.2.2"
+version = "0.2.3"
 description = "Typed secure file transfer and reproducible covert-channel measurement for IETF protocol fields."
 readme = "README.md"
 license = "Apache-2.0"

{celatim-0.2.2 → celatim-0.2.3}/src/celatim/__init__.py

@@ -361,7 +361,7 @@ from .transports import (
     extract_pcap_carriers,
 )
 
-__version__ = "0.2.2"
+__version__ = "0.2.3"
 
 __all__ = [
     "BUNDLE_SCHEMA_VERSION",

{celatim-0.2.2 → celatim-0.2.3}/src/celatim/adapter.py

@@ -549,12 +549,12 @@ def _paths_for(
                 kind=AdapterPathKind.COAP_AIOCOAP,
                 transport_kind="coap_aiocoap",
                 evidence_tier="real_daemon_path",
-                claim_status="local_aiocoap_client_server_payload_message_path",
+                claim_status="local_aiocoap_client_server_elective_option_path",
                 required_extras=("iot",),
                 records_artifact=True,
                 notes=(
                     "paired aiocoap client/server CoAP message exchange; covert bytes in "
-                    "the conforming payload, aiocoap Message codec validates"
+                    "an unknown elective option, aiocoap Message codec validates"
                 ),
             )
         )

{celatim-0.2.2 → celatim-0.2.3}/src/celatim/analysis/__init__.py

@@ -18,6 +18,13 @@ from .dataset import (
     build_records,
     write_dataset,
 )
+from .detector_metrics import (
+    DETECTOR_METRICS_SCHEMA_VERSION,
+    DetectorMetricReport,
+    LabeledDetectionScore,
+    ThresholdMetrics,
+    detector_metric_report,
+)
 from .subliminal_controls import (
     SUBLIMINAL_CONTROL_REPORT_SCHEMA_VERSION,
     build_subliminal_control_report,
@@ -27,14 +34,19 @@ __all__ = [
     "CLAIM_LEDGER_SCHEMA_VERSION",
     "CROSSHOST_PUBLIC_INDEX_SCHEMA_VERSION",
     "DATASET_SCHEMA_VERSION",
+    "DETECTOR_METRICS_SCHEMA_VERSION",
     "SUBLIMINAL_CONTROL_REPORT_SCHEMA_VERSION",
     "DatasetRecord",
+    "DetectorMetricReport",
+    "LabeledDetectionScore",
+    "ThresholdMetrics",
     "build_claim_ledger",
     "build_crosshost_public_index",
     "build_manifest",
     "build_records",
     "build_subliminal_control_report",
     "claim_count",
+    "detector_metric_report",
     "load_claim_ledger",
     "load_crosshost_public_index",
     "write_dataset",

{celatim-0.2.2 → celatim-0.2.3}/src/celatim/analysis/crosshost_evidence.py

@@ -334,11 +334,13 @@ def _public_metric_record(record: Mapping[str, Any]) -> dict[str, Any]:
         "recovered_bytes": record.get("recovered_bytes"),
         "carrier_units": record.get("carrier_units"),
         "raw_capacity_bits_per_unit": record.get("raw_capacity_bits_per_unit"),
-        "carrier_bit_efficiency": record.get("carrier_bit_efficiency"),
+        "packing_efficiency_diagnostic": record.get("packing_efficiency_diagnostic"),
         "method_wire_bytes": record.get("method_wire_bytes"),
         "method_wire_basis": record.get("method_wire_basis"),
-        "method_wire_overhead": record.get("method_wire_overhead"),
-        "payload_to_method_wire_ratio": record.get("payload_to_method_wire_ratio"),
+        "wire_expansion": record.get("wire_expansion"),
+        "useful_payload_ratio": record.get("useful_payload_ratio"),
+        "carrier_units_per_payload_byte": record.get("carrier_units_per_payload_byte"),
+        "method_wire_bytes_per_carrier_unit": record.get("method_wire_bytes_per_carrier_unit"),
         "vxlan_underlay_bytes_no_fcs": record.get("vxlan_underlay_bytes_no_fcs"),
         "payload_to_vxlan_underlay_ratio": record.get("payload_to_vxlan_underlay_ratio"),
         "timing": {
@@ -347,7 +349,8 @@ def _public_metric_record(record: Mapping[str, Any]) -> dict[str, Any]:
             "measured_window_basis": timing_map.get("measured_window_basis"),
             "observed_unit_rate_hz": timing_map.get("observed_unit_rate_hz"),
             "payload_bytes_per_s": timing_map.get("payload_bytes_per_s"),
-            "payload_bits_per_s": timing_map.get("payload_bits_per_s"),
+            "observed_recovery_rate_bps": timing_map.get("observed_recovery_rate_bps"),
+            "native_goodput_bps": timing_map.get("native_goodput_bps"),
             "scheduled_unit_rate_hz": timing_map.get("scheduled_unit_rate_hz"),
             "scheduled_duration_s": timing_map.get("scheduled_duration_s"),
         },

{celatim-0.2.2 → celatim-0.2.3}/src/celatim/analysis/crosshost_metrics.py

@@ -7,7 +7,8 @@ from collections.abc import Iterable, Mapping, Sequence
 from dataclasses import dataclass
 from typing import Any
 
-METRICS_SCHEMA_VERSION = "celatim.alice_bob_metrics.v1"
+METRICS_SCHEMA_VERSION = "celatim.alice_bob_metrics.v2"
+NATIVE_GOODPUT_CLAIM_STATUS = "native_protocol_goodput_measured"
 
 ETHERNET_HEADER_BYTES = 14
 IPV4_HEADER_BYTES = 20
@@ -74,6 +75,11 @@ def metric_record(input: MechanismMetricInput) -> dict[str, Any]:
         if input.carrier_units is not None and input.raw_capacity_bits is not None
         else None
     )
+    observed_recovery_rate_bps = (
+        _ratio(input.recovered_bytes * 8, input.measured_window_s)
+        if input.recovered_bytes is not None
+        else None
+    )
     return {
         "mechanism": input.mechanism_id,
         "suite": input.suite,
@@ -84,13 +90,15 @@ def metric_record(input: MechanismMetricInput) -> dict[str, Any]:
         "carrier_units": input.carrier_units,
         "raw_capacity_bits_per_unit": input.raw_capacity_bits,
         "carrier_capacity_bits": carrier_capacity_bits,
-        "carrier_bit_efficiency": _ratio(payload_bits, carrier_capacity_bits),
+        "packing_efficiency_diagnostic": _ratio(payload_bits, carrier_capacity_bits),
         "carrier_wire_bytes": input.carrier_wire_bytes,
         "method_wire_bytes": input.method_wire_bytes,
         "method_wire_basis": input.method_wire_basis,
         "vxlan_underlay_bytes_no_fcs": input.vxlan_underlay_bytes_no_fcs,
-        "payload_to_method_wire_ratio": _ratio(input.payload_bytes, input.method_wire_bytes),
-        "method_wire_overhead": _ratio(input.method_wire_bytes, input.payload_bytes),
+        "useful_payload_ratio": _ratio(input.payload_bytes, input.method_wire_bytes),
+        "wire_expansion": _ratio(input.method_wire_bytes, input.payload_bytes),
+        "carrier_units_per_payload_byte": _ratio(input.carrier_units, input.payload_bytes),
+        "method_wire_bytes_per_carrier_unit": _ratio(input.method_wire_bytes, input.carrier_units),
         "payload_to_vxlan_underlay_ratio": _ratio(
             input.payload_bytes, input.vxlan_underlay_bytes_no_fcs
         ),
@@ -102,9 +110,10 @@ def metric_record(input: MechanismMetricInput) -> dict[str, Any]:
             "scheduled_duration_s": input.scheduled_duration_s,
             "observed_unit_rate_hz": _ratio(input.carrier_units, input.measured_window_s),
             "payload_bytes_per_s": _ratio(input.recovered_bytes, input.measured_window_s),
-            "payload_bits_per_s": (
-                _ratio(input.recovered_bytes * 8, input.measured_window_s)
-                if input.recovered_bytes is not None
+            "observed_recovery_rate_bps": observed_recovery_rate_bps,
+            "native_goodput_bps": (
+                observed_recovery_rate_bps
+                if input.timing_claim_status == NATIVE_GOODPUT_CLAIM_STATUS
                 else None
             ),
             "claim_status": input.timing_claim_status,
@@ -123,7 +132,7 @@ def metrics_summary(
     measured = [
         record
         for record in pass_records
-        if _nested_number(record, ("timing", "payload_bits_per_s")) is not None
+        if _nested_number(record, ("timing", "observed_recovery_rate_bps")) is not None
     ]
     return {
         "schema_version": METRICS_SCHEMA_VERSION,
@@ -134,23 +143,27 @@ def metrics_summary(
         "timing_claim_status_counts": _counts(
             _nested_value(record, ("timing", "claim_status")) for record in records
         ),
-        "method_wire_overhead": _stats(
-            _number(record.get("method_wire_overhead")) for record in pass_records
+        "wire_expansion": _stats(_number(record.get("wire_expansion")) for record in pass_records),
+        "useful_payload_ratio": _stats(
+            _number(record.get("useful_payload_ratio")) for record in pass_records
+        ),
+        "carrier_units_per_payload_byte": _stats(
+            _number(record.get("carrier_units_per_payload_byte")) for record in pass_records
         ),
-        "payload_to_method_wire_ratio": _stats(
-            _number(record.get("payload_to_method_wire_ratio")) for record in pass_records
+        "observed_recovery_rate_bps": _stats(
+            _nested_number(record, ("timing", "observed_recovery_rate_bps")) for record in measured
         ),
-        "payload_bits_per_s": _stats(
-            _nested_number(record, ("timing", "payload_bits_per_s")) for record in measured
+        "native_goodput_bps": _stats(
+            _nested_number(record, ("timing", "native_goodput_bps")) for record in measured
         ),
         "observed_unit_rate_hz": _stats(
             _nested_number(record, ("timing", "observed_unit_rate_hz")) for record in measured
         ),
-        "fastest_payload_bits_per_s": _extreme(
-            measured, ("timing", "payload_bits_per_s"), reverse=True
+        "fastest_observed_recovery_rate_bps": _extreme(
+            measured, ("timing", "observed_recovery_rate_bps"), reverse=True
         ),
-        "slowest_payload_bits_per_s": _extreme(
-            measured, ("timing", "payload_bits_per_s"), reverse=False
+        "slowest_observed_recovery_rate_bps": _extreme(
+            measured, ("timing", "observed_recovery_rate_bps"), reverse=False
         ),
     }
 
@@ -191,10 +204,13 @@ def _counts(values: Iterable[Any]) -> dict[str, int]:
 def _stats(values: Iterable[float | None]) -> dict[str, float | None]:
     series = sorted(value for value in values if value is not None)
     if not series:
-        return {"min": None, "median": None, "max": None}
+        return {"n": 0, "min": None, "q1": None, "median": None, "q3": None, "max": None}
     return {
+        "n": len(series),
         "min": series[0],
+        "q1": _quantile(series, 0.25),
         "median": _median(series),
+        "q3": _quantile(series, 0.75),
         "max": series[-1],
     }
 
@@ -206,6 +222,16 @@ def _median(series: Sequence[float]) -> float:
     return (series[midpoint - 1] + series[midpoint]) / 2.0
 
 
+def _quantile(series: Sequence[float], probability: float) -> float:
+    position = (len(series) - 1) * probability
+    lower = math.floor(position)
+    upper = math.ceil(position)
+    if lower == upper:
+        return series[lower]
+    fraction = position - lower
+    return series[lower] + fraction * (series[upper] - series[lower])
+
+
 def _extreme(
     records: Sequence[Mapping[str, Any]],
     path: Sequence[str],
@@ -227,6 +253,7 @@ def _extreme(
 
 __all__ = [
     "METRICS_SCHEMA_VERSION",
+    "NATIVE_GOODPUT_CLAIM_STATUS",
     "MechanismMetricInput",
     "carrier_lengths_from_envelope",
     "metric_record",

celatim-0.2.3/src/celatim/analysis/detector_metrics.py

@@ -0,0 +1,192 @@
+"""Threshold-sweep metrics for labeled detector scores."""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Iterable
+from dataclasses import dataclass
+from itertools import pairwise
+from typing import Any
+
+DETECTOR_METRICS_SCHEMA_VERSION = "celatim.detector_metrics.v1"
+
+
+@dataclass(frozen=True)
+class LabeledDetectionScore:
+    label: bool
+    score: float
+
+    def __post_init__(self) -> None:
+        if not math.isfinite(self.score):
+            raise ValueError("detector score must be finite")
+
+
+@dataclass(frozen=True)
+class ThresholdMetrics:
+    threshold: float
+    true_positive: int
+    false_positive: int
+    true_negative: int
+    false_negative: int
+    true_positive_rate: float
+    false_positive_rate: float
+    precision: float | None
+    recall: float
+    tpr_wilson95: tuple[float, float]
+    fpr_wilson95: tuple[float, float]
+    prevalence_adjusted_precision: dict[str, float | None]
+
+    def to_json(self) -> dict[str, Any]:
+        return {
+            "threshold": self.threshold,
+            "true_positive": self.true_positive,
+            "false_positive": self.false_positive,
+            "true_negative": self.true_negative,
+            "false_negative": self.false_negative,
+            "true_positive_rate": self.true_positive_rate,
+            "false_positive_rate": self.false_positive_rate,
+            "precision": self.precision,
+            "recall": self.recall,
+            "tpr_wilson95": list(self.tpr_wilson95),
+            "fpr_wilson95": list(self.fpr_wilson95),
+            "prevalence_adjusted_precision": self.prevalence_adjusted_precision,
+        }
+
+
+@dataclass(frozen=True)
+class DetectorMetricReport:
+    positive_count: int
+    negative_count: int
+    roc_auc: float
+    average_precision: float
+    prevalence_assumptions: tuple[float, ...]
+    thresholds: tuple[ThresholdMetrics, ...]
+
+    def to_json(self) -> dict[str, Any]:
+        return {
+            "schema_version": DETECTOR_METRICS_SCHEMA_VERSION,
+            "positive_count": self.positive_count,
+            "negative_count": self.negative_count,
+            "roc_auc": self.roc_auc,
+            "average_precision": self.average_precision,
+            "prevalence_assumptions": list(self.prevalence_assumptions),
+            "thresholds": [row.to_json() for row in self.thresholds],
+        }
+
+
+def detector_metric_report(
+    scores: Iterable[LabeledDetectionScore],
+    *,
+    prevalence_assumptions: tuple[float, ...] = (0.0001, 0.001, 0.01),
+) -> DetectorMetricReport:
+    observations = tuple(scores)
+    positives = sum(observation.label for observation in observations)
+    negatives = len(observations) - positives
+    if positives == 0 or negatives == 0:
+        raise ValueError("detector evaluation requires positive and negative observations")
+    if any(not 0 < prevalence < 1 for prevalence in prevalence_assumptions):
+        raise ValueError("prevalence assumptions must be between zero and one")
+    unique_scores = sorted({observation.score for observation in observations}, reverse=True)
+    thresholds = (math.inf, *unique_scores, -math.inf)
+    rows = tuple(
+        _threshold_metrics(
+            observations,
+            threshold,
+            prevalence_assumptions=prevalence_assumptions,
+        )
+        for threshold in thresholds
+    )
+    roc_points = sorted(
+        ((row.false_positive_rate, row.true_positive_rate) for row in rows),
+        key=lambda point: (point[0], point[1]),
+    )
+    precision_rows = [row for row in rows if row.precision is not None]
+    precision_rows.sort(key=lambda row: row.recall)
+    return DetectorMetricReport(
+        positive_count=positives,
+        negative_count=negatives,
+        roc_auc=_trapezoid(roc_points),
+        average_precision=_step_average_precision(precision_rows),
+        prevalence_assumptions=prevalence_assumptions,
+        thresholds=rows,
+    )
+
+
+def _threshold_metrics(
+    scores: tuple[LabeledDetectionScore, ...],
+    threshold: float,
+    *,
+    prevalence_assumptions: tuple[float, ...],
+) -> ThresholdMetrics:
+    true_positive = sum(item.label and item.score >= threshold for item in scores)
+    false_positive = sum(not item.label and item.score >= threshold for item in scores)
+    positive_count = sum(item.label for item in scores)
+    negative_count = len(scores) - positive_count
+    false_negative = positive_count - true_positive
+    true_negative = negative_count - false_positive
+    tpr = true_positive / positive_count
+    fpr = false_positive / negative_count
+    predicted_positive = true_positive + false_positive
+    precision = true_positive / predicted_positive if predicted_positive else None
+    adjusted = {
+        f"{prevalence:g}": _prevalence_precision(tpr, fpr, prevalence)
+        for prevalence in prevalence_assumptions
+    }
+    return ThresholdMetrics(
+        threshold=threshold,
+        true_positive=true_positive,
+        false_positive=false_positive,
+        true_negative=true_negative,
+        false_negative=false_negative,
+        true_positive_rate=tpr,
+        false_positive_rate=fpr,
+        precision=precision,
+        recall=tpr,
+        tpr_wilson95=_wilson95(true_positive, positive_count),
+        fpr_wilson95=_wilson95(false_positive, negative_count),
+        prevalence_adjusted_precision=adjusted,
+    )
+
+
+def _prevalence_precision(tpr: float, fpr: float, prevalence: float) -> float | None:
+    denominator = tpr * prevalence + fpr * (1 - prevalence)
+    return tpr * prevalence / denominator if denominator else None
+
+
+def _wilson95(successes: int, trials: int) -> tuple[float, float]:
+    z = 1.959963984540054
+    proportion = successes / trials
+    denominator = 1 + z * z / trials
+    center = (proportion + z * z / (2 * trials)) / denominator
+    margin = (
+        z
+        * math.sqrt(proportion * (1 - proportion) / trials + z * z / (4 * trials * trials))
+        / denominator
+    )
+    return max(0.0, center - margin), min(1.0, center + margin)
+
+
+def _trapezoid(points: list[tuple[float, float]]) -> float:
+    area = 0.0
+    for left, right in pairwise(points):
+        area += (right[0] - left[0]) * (left[1] + right[1]) / 2
+    return area
+
+
+def _step_average_precision(rows: list[ThresholdMetrics]) -> float:
+    area = 0.0
+    previous_recall = 0.0
+    for row in rows:
+        if row.recall > previous_recall and row.precision is not None:
+            area += (row.recall - previous_recall) * row.precision
+            previous_recall = row.recall
+    return area
+
+
+__all__ = [
+    "DETECTOR_METRICS_SCHEMA_VERSION",
+    "DetectorMetricReport",
+    "LabeledDetectionScore",
+    "ThresholdMetrics",
+    "detector_metric_report",
+]