bspe 0.1.0__py3-none-any.whl

bspe/__init__.py

@@ -0,0 +1,212 @@
+"""Reusable core for binary hidden-Markov predictive entropy analysis."""
+
+from bspe.analysis import analyze_sequence
+from bspe.batch_parsing import (
+    CsvBatchColumns,
+    parse_csv_batch,
+    parse_manual_batch,
+    parse_txt_batch,
+)
+from bspe.domain import BinaryHMM, BinaryLabels, SequenceAnalysis
+from bspe.markov_batch_serialization import markov_batch_summary_csv
+from bspe.markov_serialization import (
+    markov_model_json,
+    markov_sequence_csv,
+)
+from bspe.markov_types import (
+    MarkovBatchAnalysis,
+    MarkovEstimation,
+    MarkovModel,
+    MarkovPredictionMode,
+    MarkovResultScope,
+)
+from bspe.methods.hmm import HMMBatchAnalysis, analyze_hmm
+from bspe.methods.markov import (
+    analyze_markov,
+    analyze_markov_per_sequence,
+    fit_markov,
+    predict_markov,
+)
+from bspe.methods.shannon import (
+    ShannonBatchAnalysis,
+    ShannonPrefixResult,
+    analyze_shannon,
+)
+from bspe.methods.vmm import analyze_vmm, analyze_vmm_per_sequence, fit_vmm
+from bspe.parsing import parse_sequence
+from bspe.records import (
+    BinarySequence,
+    SequenceDataset,
+    SequenceId,
+    SequenceRecord,
+)
+from bspe.stimulus_generation import generate_candidate_records
+from bspe.stimulus_matching import (
+    create_complement_candidates,
+    match_stimuli,
+)
+from bspe.stimulus_metrics import sequence_metrics
+from bspe.stimulus_search import evaluate_candidate, search_stimuli
+from bspe.stimulus_search_csv import (
+    stimulus_candidate_csv,
+    stimulus_experiment_csv,
+    stimulus_scientific_csv,
+)
+from bspe.stimulus_search_json import stimulus_generator_config_json
+from bspe.stimulus_search_results import (
+    ConstraintViolation,
+    MatchingResult,
+    MatchTolerances,
+    MetricSummary,
+    SearchPartialReason,
+    SearchResult,
+    SearchStatus,
+    SequenceMetrics,
+    StimulusCandidate,
+    StimulusPair,
+    TargetCongruency,
+    ValidationReport,
+)
+from bspe.stimulus_search_types import (
+    InclusiveRange,
+    InvalidStimulusSearchConfigurationError,
+    PredictedSymbol,
+    PreferenceMetric,
+    SoftPreference,
+    StimulusConstraints,
+    StimulusId,
+    StimulusSearchConfig,
+)
+from bspe.stimulus_targets import assign_targets, validate_stimuli
+from bspe.vmm_serialization import (
+    vmm_context_evidence_csv,
+    vmm_context_model_json,
+    vmm_evaluation_csv,
+)
+from bspe.vmm_types import (
+    AdditiveSmoothing,
+    InvalidVMMConfigurationError,
+    KTSmoothing,
+    MLESmoothing,
+    VMMAnalysis,
+    VMMConfig,
+    VMMContextCount,
+    VMMDepthAnalysis,
+    VMMDepthStatus,
+    VMMModel,
+    VMMRecordAnalysis,
+    VMMResultScope,
+    VMMSmoothing,
+)
+from bspe.workbench import (
+    AnalysisMethod,
+    HMMAnalysisRequest,
+    MarkovAnalysisRequest,
+    MethodComparison,
+    ShannonAnalysisRequest,
+    VMMAnalysisRequest,
+    analyze_dataset,
+    compare_methods,
+)
+
+__all__ = [
+    "AdditiveSmoothing",
+    "AnalysisMethod",
+    "BinaryHMM",
+    "BinaryLabels",
+    "BinarySequence",
+    "ConstraintViolation",
+    "CsvBatchColumns",
+    "HMMAnalysisRequest",
+    "HMMBatchAnalysis",
+    "InclusiveRange",
+    "InvalidStimulusSearchConfigurationError",
+    "InvalidVMMConfigurationError",
+    "KTSmoothing",
+    "MLESmoothing",
+    "MarkovAnalysisRequest",
+    "MarkovBatchAnalysis",
+    "MarkovEstimation",
+    "MarkovModel",
+    "MarkovPredictionMode",
+    "MarkovResultScope",
+    "MatchTolerances",
+    "MatchingResult",
+    "MethodComparison",
+    "MetricSummary",
+    "PredictedSymbol",
+    "PreferenceMetric",
+    "SearchPartialReason",
+    "SearchResult",
+    "SearchStatus",
+    "SequenceAnalysis",
+    "SequenceDataset",
+    "SequenceId",
+    "SequenceMetrics",
+    "SequenceRecord",
+    "ShannonAnalysisRequest",
+    "ShannonBatchAnalysis",
+    "ShannonPrefixResult",
+    "SoftPreference",
+    "StimulusCandidate",
+    "StimulusConstraints",
+    "StimulusId",
+    "StimulusPair",
+    "StimulusSearchConfig",
+    "TargetCongruency",
+    "VMMAnalysis",
+    "VMMAnalysisRequest",
+    "VMMConfig",
+    "VMMContextCount",
+    "VMMDepthAnalysis",
+    "VMMDepthStatus",
+    "VMMModel",
+    "VMMRecordAnalysis",
+    "VMMResultScope",
+    "VMMSmoothing",
+    "ValidationReport",
+    "analyze_dataset",
+    "analyze_hmm",
+    "analyze_markov",
+    "analyze_markov_per_sequence",
+    "analyze_sequence",
+    "analyze_shannon",
+    "analyze_vmm",
+    "analyze_vmm_per_sequence",
+    "assign_targets",
+    "assign_outcomes",
+    "TargetAssignmentMode",
+    "PresentationMetadata",
+    "apply_presentation_metadata",
+    "compare_methods",
+    "create_complement_candidates",
+    "evaluate_candidate",
+    "fit_markov",
+    "fit_vmm",
+    "generate_candidate_records",
+    "markov_batch_summary_csv",
+    "markov_model_json",
+    "markov_sequence_csv",
+    "match_stimuli",
+    "parse_csv_batch",
+    "parse_manual_batch",
+    "parse_sequence",
+    "parse_txt_batch",
+    "predict_markov",
+    "search_stimuli",
+    "sequence_metrics",
+    "stimulus_candidate_csv",
+    "stimulus_experiment_csv",
+    "stimulus_generator_config_json",
+    "stimulus_scientific_csv",
+    "validate_stimuli",
+    "vmm_context_evidence_csv",
+    "vmm_context_model_json",
+    "vmm_evaluation_csv",
+]
+
+from bspe.stimulus_metadata import (
+    PresentationMetadata,
+    apply_presentation_metadata,
+)
+from bspe.stimulus_targets import TargetAssignmentMode, assign_outcomes

bspe/_vmm_serialization_common.py

@@ -0,0 +1,176 @@
+"""Shared provenance and status values for VMM raw exports."""
+
+import hashlib
+import json
+import math
+from dataclasses import dataclass
+from typing import Final
+
+from bspe.records import BinarySequence, SequenceDataset, SequenceRecord
+from bspe.vmm_types import (
+    AdditiveSmoothing,
+    KTSmoothing,
+    MLESmoothing,
+    VMMAnalysis,
+    VMMDepthAnalysis,
+    VMMDepthStatus,
+    VMMRecordAnalysis,
+    VMMSmoothing,
+)
+
+EXPERIMENTAL_STATUS: Final = "experimental"
+EXPERIMENTAL_NOTICE: Final = (
+    "Experimental raw VMM artifact; retain its schema and source details when reused."
+)
+MLE_UNAVAILABLE_REASON: Final = (
+    "MLE unavailable: unseen context has no occurrences in the training dataset."
+)
+CONFIGURED_DEPTH_SELECTION: Final = "deepest_supported_suffix"
+DATASET_ROLE: Final = "training"
+type JsonMetric = float | str | None
+_ESTIMATION_RULES: Final = {
+    KTSmoothing: "krichevsky_trofimov",
+    MLESmoothing: "maximum_likelihood",
+    AdditiveSmoothing: "additive_smoothing",
+}
+_SUPPORT_STATUSES: Final = {
+    VMMDepthStatus.ACCEPTED: ("accepted", "not_sparse"),
+    VMMDepthStatus.LOW_SUPPORT: ("low_support", "sparse"),
+    VMMDepthStatus.UNAVAILABLE: ("unavailable", "unavailable"),
+}
+_UNAVAILABLE_REASONS: Final = {
+    KTSmoothing: "Unseen context has no occurrences in the training dataset.",
+    MLESmoothing: MLE_UNAVAILABLE_REASON,
+    AdditiveSmoothing: "Unseen context has no occurrences in the training dataset.",
+}
+
+
+@dataclass(frozen=True, slots=True)
+class ExportContext:
+    """Inputs and stable identity shared by all three artifacts."""
+
+    analysis: VMMAnalysis
+    dataset: SequenceDataset
+    training_identifier: str
+
+
+@dataclass(frozen=True, slots=True)
+class RecordPair:
+    """One analyzed record paired with its ordered source stimulus."""
+
+    analysis: VMMRecordAnalysis
+    source: SequenceRecord
+    source_order: int
+
+
+def export_context(analysis: VMMAnalysis, dataset: SequenceDataset) -> ExportContext:
+    """Build shared export provenance from immutable analysis inputs."""
+    identity = (
+        ("observable_labels", dataset.labels.observables),
+        (
+            "records",
+            tuple((record.sequence_id, record.sequence) for record in dataset.records),
+        ),
+    )
+    canonical = json.dumps(
+        identity,
+        ensure_ascii=False,
+        allow_nan=False,
+        separators=(",", ":"),
+    ).encode("utf-8", errors="strict")
+    digest = hashlib.sha256(canonical).hexdigest()
+    return ExportContext(analysis, dataset, f"sha256:{digest}")
+
+
+def record_pairs(context: ExportContext) -> tuple[RecordPair, ...]:
+    """Pair analysis records to source records without changing source order."""
+    return tuple(
+        RecordPair(analyzed, source, source_order)
+        for source_order, (analyzed, source) in enumerate(
+            zip(context.analysis.records, context.dataset.records, strict=True),
+            start=1,
+        )
+    )
+
+
+def estimation_rule(analysis: VMMAnalysis) -> str:
+    """Name the configured estimator without inferring from alpha alone."""
+    return _ESTIMATION_RULES[type(analysis.config.smoothing)]
+
+
+def labeled_sequence(
+    sequence: BinarySequence,
+    labels: tuple[str, str],
+) -> tuple[str, ...]:
+    """Map internal observable indices to retained source labels."""
+    return tuple(labels[index] for index in sequence)
+
+
+def csv_data(text: str) -> str:
+    """Neutralize spreadsheet formula prefixes in untrusted text cells."""
+    return f"'{text}" if text.lstrip().startswith(("=", "+", "-", "@")) else text
+
+
+def support_status(status: VMMDepthStatus) -> tuple[str, str]:
+    """Return separate support and sparse statuses for one examined context."""
+    return _SUPPORT_STATUSES[status]
+
+
+def context_reason(row: VMMDepthAnalysis, smoothing: VMMSmoothing) -> str | None:
+    """State why an examined context was rejected when a reason applies."""
+    reasons = {
+        VMMDepthStatus.ACCEPTED: None,
+        VMMDepthStatus.LOW_SUPPORT: (
+            "Context support is below the configured minimum."
+        ),
+        VMMDepthStatus.UNAVAILABLE: _UNAVAILABLE_REASONS[type(smoothing)],
+    }
+    return reasons[row.status]
+
+
+def backoff_selection(record: VMMRecordAnalysis) -> str:
+    """Name whether the requested full suffix was selected or backed off."""
+    if record.effective_context_depth is None:
+        return "unavailable"
+    if record.effective_context_depth == len(record.sequence):
+        return "requested_depth_selected"
+    return "backed_off_to_shorter_suffix"
+
+
+def backoff_reason(
+    record: VMMRecordAnalysis,
+    smoothing: VMMSmoothing,
+) -> str | None:
+    """Explain the deepest requested suffix outcome from retained evidence."""
+    if record.effective_context_depth == len(record.sequence):
+        return None
+    reason = context_reason(record.depth_rows[-1], smoothing)
+    return reason or "No context meets the configured minimum support."
+
+
+def selected_row(record: VMMRecordAnalysis) -> VMMDepthAnalysis | None:
+    """Return the retained evidence row selected for final prediction."""
+    return next(
+        (
+            row
+            for row in record.depth_rows
+            if row.depth == record.effective_context_depth
+        ),
+        None,
+    )
+
+
+def evaluation_status(pair: RecordPair) -> str:
+    """Label training-record target assessment without claiming held-out data."""
+    if pair.source.actual_target_index is None:
+        return "Not supplied"
+    if pair.analysis.target_assessment is None:
+        return "In-sample evaluation unavailable"
+    return "In-sample evaluation, not held out"
+
+
+def json_metric(value: float | None) -> JsonMetric:
+    """Represent defined infinite surprisal without non-standard JSON numbers."""
+    if value is None or math.isfinite(value):
+        return value
+    return "infinity" if value > 0.0 else "-infinity"

bspe/_vmm_serialization_csv.py

@@ -0,0 +1,222 @@
+"""High-precision CSV rows for experimental VMM raw artifacts."""
+
+from typing import Final
+
+from bspe._vmm_serialization_common import (
+    CONFIGURED_DEPTH_SELECTION,
+    DATASET_ROLE,
+    EXPERIMENTAL_NOTICE,
+    EXPERIMENTAL_STATUS,
+    ExportContext,
+    RecordPair,
+    backoff_reason,
+    backoff_selection,
+    context_reason,
+    csv_data,
+    estimation_rule,
+    evaluation_status,
+    labeled_sequence,
+    record_pairs,
+    selected_row,
+    support_status,
+)
+from bspe.information import surprisal
+from bspe.markov_csv import CsvCell, markov_csv_text
+from bspe.records import BinarySequence
+from bspe.vmm_types import VMMDepthAnalysis
+
+COLUMNS: Final = (
+    "artifact_name",
+    "experimental_status",
+    "experimental_notice",
+    "method",
+    "dataset_role",
+    "training_dataset_identifier",
+    "evaluation_dataset_identifier",
+    "record_identifier",
+    "source_order",
+    "observable_A_label",
+    "observable_B_label",
+    "sequence_stimulus",
+    "consumed_prefix_stimulus",
+    "sequence_length",
+    "consumed_prefix_depth",
+    "displayed_context",
+    "requested_depth",
+    "actual_depth",
+    "workflow",
+    "result_scope",
+    "configured_depth_selection",
+    "estimation_rule",
+    "smoothing_alpha",
+    "suffix_backoff_selection",
+    "suffix_backoff_reason",
+    "context_occurrence_count",
+    "next_A_count",
+    "next_B_count",
+    "support_rule",
+    "support_status",
+    "sparse_status",
+    "next_A_probability",
+    "next_B_probability",
+    "predictive_entropy_bits",
+    "observed_target",
+    "target_probability",
+    "target_surprisal_bits",
+    "target_classification",
+    "evaluation_status",
+)
+
+
+def context_evidence_csv(context: ExportContext) -> str:
+    """Serialize every retained depth row in source and ascending-depth order."""
+    rows = tuple(
+        _evidence_row(context, pair, row)
+        for pair in record_pairs(context)
+        for row in pair.analysis.depth_rows
+    )
+    return markov_csv_text(COLUMNS, rows)
+
+
+def evaluation_csv(context: ExportContext) -> str:
+    """Serialize one final evaluation row per ordered training record."""
+    rows = tuple(_evaluation_row(context, pair) for pair in record_pairs(context))
+    return markov_csv_text(COLUMNS, rows)
+
+
+def _evidence_row(
+    context: ExportContext,
+    pair: RecordPair,
+    row: VMMDepthAnalysis,
+) -> tuple[CsvCell, ...]:
+    record = pair.analysis
+    labels = context.dataset.labels.observables
+    selected = row.depth == record.effective_context_depth
+    support, sparse = support_status(row.status)
+    target = record.target_assessment if selected else None
+    target_index = pair.source.actual_target_index
+    target_probability = (
+        None
+        if target_index is None
+        else (row.probability_a, row.probability_b)[target_index]
+    )
+    if target_index is None:
+        row_evaluation_status = "Not supplied"
+    elif target_probability is None:
+        row_evaluation_status = "In-sample evaluation unavailable"
+    else:
+        row_evaluation_status = "In-sample evaluation, not held out"
+    if record.effective_context_depth is None:
+        selection = "no_context_selected"
+    elif selected:
+        selection = "selected"
+    elif row.depth > record.effective_context_depth:
+        selection = "rejected_for_backoff"
+    else:
+        selection = "not_selected_shorter_context"
+    return (
+        *_provenance(context, pair, "Context evidence export"),
+        _context_text(row.matched_suffix, labels),
+        row.depth,
+        record.effective_context_depth,
+        "variable_order_markov",
+        context.analysis.result_scope.value,
+        CONFIGURED_DEPTH_SELECTION,
+        estimation_rule(context.analysis),
+        context.analysis.config.smoothing.alpha,
+        selection,
+        context_reason(row, context.analysis.config.smoothing),
+        row.support,
+        row.count_a,
+        row.count_b,
+        f"minimum_support={context.analysis.config.minimum_support}",
+        support,
+        sparse,
+        row.probability_a,
+        row.probability_b,
+        row.predictive_entropy_bits,
+        None if target_index is None else csv_data(labels[target_index]),
+        target_probability,
+        None if target_probability is None else surprisal(target_probability),
+        None if target is None else target.classification.value,
+        row_evaluation_status,
+    )
+
+
+def _evaluation_row(
+    context: ExportContext,
+    pair: RecordPair,
+) -> tuple[CsvCell, ...]:
+    record = pair.analysis
+    labels = context.dataset.labels.observables
+    selected = selected_row(record)
+    support, sparse = (
+        support_status(selected.status)
+        if selected is not None
+        else ("unavailable", "unavailable")
+    )
+    target = record.target_assessment
+    target_index = pair.source.actual_target_index
+    return (
+        *_provenance(context, pair, "Evaluation export"),
+        (
+            None
+            if record.context_used is None
+            else _context_text(record.context_used, labels)
+        ),
+        len(record.sequence),
+        record.effective_context_depth,
+        "variable_order_markov",
+        context.analysis.result_scope.value,
+        CONFIGURED_DEPTH_SELECTION,
+        estimation_rule(context.analysis),
+        context.analysis.config.smoothing.alpha,
+        backoff_selection(record),
+        backoff_reason(record, context.analysis.config.smoothing),
+        None if selected is None else selected.support,
+        None if selected is None else selected.count_a,
+        None if selected is None else selected.count_b,
+        f"minimum_support={context.analysis.config.minimum_support}",
+        support,
+        sparse,
+        record.probability_a,
+        record.probability_b,
+        record.predictive_entropy_bits,
+        None if target_index is None else csv_data(labels[target_index]),
+        None if target is None else target.probability,
+        None if target is None else target.surprisal_bits,
+        None if target is None else target.classification.value,
+        evaluation_status(pair),
+    )
+
+
+def _provenance(
+    context: ExportContext,
+    pair: RecordPair,
+    artifact_name: str,
+) -> tuple[CsvCell, ...]:
+    labels = context.dataset.labels.observables
+    stimulus = csv_data(",".join(labeled_sequence(pair.analysis.sequence, labels)))
+    return (
+        artifact_name,
+        EXPERIMENTAL_STATUS,
+        EXPERIMENTAL_NOTICE,
+        "vmm",
+        DATASET_ROLE,
+        context.training_identifier,
+        None,
+        csv_data(str(pair.source.sequence_id)),
+        pair.source_order,
+        csv_data(labels[0]),
+        csv_data(labels[1]),
+        stimulus,
+        stimulus,
+        len(pair.analysis.sequence),
+        len(pair.analysis.sequence),
+    )
+
+
+def _context_text(context: BinarySequence, labels: tuple[str, str]) -> str:
+    if not context:
+        return "Order 0 (no suffix)"
+    return csv_data(",".join(labeled_sequence(context, labels)))