ZaksPhysicsLibrary 1.3.0__tar.gz → 1.5.0__tar.gz

{zaksphysicslibrary-1.3.0/ZaksPhysicsLibrary.egg-info → zaksphysicslibrary-1.5.0}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: ZaksPhysicsLibrary
-Version: 1.3.0
-Summary: Data processing and analysis library for TDT, Oxysoft NIRS, and Terranova EFNMR lab data
+Version: 1.5.0
+Summary: Data processing and analysis library for TDT, Oxysoft NIRS, Terranova EFNMR lab data, and grouped-text-field studies
 Author: zakgm2
 License-Expression: MIT
 Project-URL: Homepage, https://github.com/zakgm2/PhysicsLibrary
 Project-URL: Repository, https://github.com/zakgm2/PhysicsLibrary
 Project-URL: Changelog, https://github.com/zakgm2/PhysicsLibrary/blob/main/CHANGELOG.md
-Keywords: physics,neuroscience,fibre-photometry,NIRS,TDT,EFNMR,signal-processing
+Keywords: physics,neuroscience,fibre-photometry,NIRS,TDT,EFNMR,signal-processing,text-analysis,nlp
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -21,6 +21,9 @@ License-File: LICENSE
 Requires-Dist: numpy
 Requires-Dist: scipy
 Requires-Dist: tdt
+Requires-Dist: pandas
+Requires-Dist: sentence-transformers
+Requires-Dist: statsmodels
 Dynamic: license-file
 
 # PhysicsLibrary
@@ -38,6 +41,8 @@ Data processing and analysis library backing [PhysicsAnalysis](https://github.co
   - Generic **Excel / CSV / TSV / plain text**, with automatic sub-table detection for side-by-side data layouts on one sheet
 - **Processes signals** — bleach correction, denoising, Z-score PETH slicing, FFT with peak annotation, slope/segment analysis
 - **Fits curves** — linear, single/double exponential, exponential rise, Gaussian, sinusoidal, and a photon-entanglement visibility model, all via `scipy.optimize.curve_fit`
+- **Analyses text field studies** — one JSON file per subject with several free-text fields; pick any pair of fields to compare directly (e.g. does the answer to one question track another for the same subject). Word counts, data-quality flagging, sentence-transformers embeddings, an optional delta-vector magnitude between two fields, and an optional paired-similarity metric per pair with a permutation test and a word-count confound check. Domain-agnostic — field names and which pairs to compare are supplied by the caller, nothing is hardcoded to one study
+- **Validates the similarity metric statistically** — Benjamini-Hochberg FDR-corrected permutation-test p-values, Cohen's d effect size, a word-count-controlled OLS regression (statsmodels), a bootstrap confidence interval on the mean, and a leave-one-out sensitivity check, one row per field pair, with docstrings explaining what each statistic means
 
 ---
 
@@ -52,6 +57,8 @@ PhysicsLibrary/
   processing_TDT.py          TDT tank reading, bleach correction, denoising, event markers
   analysis.py                 PETH/Z-score, FFT, slope segments, curve-fit runner
   models.py                    Parametric model functions for curve fitting
+  text_field_study.py           Grouped-text-field study pipeline (embeddings, delta vector, paired similarity)
+  field_study_validation.py      Statistical validation for the paired-similarity metric (permutation test, Cohen's d, regression, bootstrap CI, leave-one-out)
   loaders/
     tdt_loader.py               Wraps processing_TDT into a Dataset
     oxysoft_loader.py            Oxysoft .txt parsing (folder + single-file) into a Dataset
@@ -77,7 +84,8 @@ git+https://github.com/zakgm2/PhysicsLibrary.git
 ### Requirements
 
 - Python 3.10+
-- `numpy`, `scipy`, `tdt` (installed automatically)
+- `numpy`, `scipy`, `tdt`, `pandas`, `sentence-transformers`, `statsmodels` (installed automatically)
+- `sentence-transformers` pulls in `torch`/`transformers` as transitive dependencies — a genuinely heavy install (hundreds of MB) if you only need the signal-processing side; only actually loaded when you call `embed_text_fields`/`run_field_study_pipeline`
 - `openpyxl` — only needed for `.xlsx`/`.xls` files; imported lazily with a clear error if missing when you actually try to load Excel
 
 ---
@@ -125,6 +133,34 @@ result = pl.fit_model_to_segment(x_seg, y_seg, pl.single_exponential_model, p0_f
 result["popt"], result["r2"], result["y_fit"]
 ```
 
+```python
+# Text field study — one JSON file per subject, e.g. P-0001.json. Pick
+# pairs of fields to compare directly; no grouping concept needed.
+fields = pl.peek_fields(folder_path)              # see what fields exist before picking pairs
+df = pl.run_field_study_pipeline(
+    folder_path,
+    text_fields=["q1", "q2", "q3", "q4"],
+    delta_pair=("q1", "q2"),                        # optional: how much did q2 change from q1
+    paired_fields=[("q1", "q3", "pair1")],           # optional: does q1 track q3
+)
+# df has one row per subject: wordcount_<field>, low_quality_<field>, delta_magnitude,
+# sim_<pair>, pvalue_<pair>, effect_size_<pair>, wc_confound_r_<pair>, ...
+```
+
+```python
+# Statistical validation of the paired-similarity metric — one row per pair
+summary = pl.run_validation_pipeline(
+    folder_path,
+    text_fields=["q1", "q3"],
+    paired_fields=[("q1", "q3", "pair1")],
+)
+# summary: p_value, p_value_fdr, cohens_d, wc_coef_a/b + wc_pvalue_a/b,
+# regression_r_squared, ci_lower/ci_upper, n_flagged_loo, flagged_participant_ids
+```
+
+See [FIELD_STUDY_METHODOLOGY.md](FIELD_STUDY_METHODOLOGY.md) for why each statistic in the
+validation step is a sound, standard technique — useful if anyone asks.
+
 See [CHANGELOG.md](CHANGELOG.md) for the version history.
 
 ---
@@ -135,8 +171,10 @@ Everything importable from `PhysicsLibrary` directly:
 
 | Category | Names |
 |----------|-------|
-| Format detection | `choose_file`, `detect_format`, `detect_format_file`, `DataFormat`, `Dataset` |
+| Format detection | `detect_format`, `detect_format_file`, `DataFormat`, `Dataset` |
 | Loading | `load_dataset`, `load_dataset_file`, `load_any_file`, `load_pt2` |
 | TDT processing | `process_tdt_folder`, `validate_tdt_folder`, `get_tdt_struct`, `get_plot_data`, `correct_bleaching`, `denoise_signal`, `get_event_markers` |
 | Analysis | `get_zscore_slice`, `smooth_signal`, `bin_for_heatmap`, `compute_fft_slice`, `annotate_fft_peaks`, `compute_slope_segment`, `fit_model_to_segment` |
 | Curve fit models | `linear_model`, `single_exponential_model`, `exponential_rise_model`, `double_exponential_model`, `gaussian_model`, `sinusoidal_model`, `visibility_model` |
+| Text field study | `run_field_study_pipeline`, `load_field_study_folder`, `peek_fields`, `flag_low_quality`, `embed_text_fields`, `compute_delta_vector`, `compute_paired_similarity`, `permutation_test_similarity`, `wordcount_confound_check` |
+| Field study validation | `run_validation_pipeline`, `build_validation_summary`, `cohens_d`, `benjamini_hochberg`, `wordcount_controlled_regression`, `bootstrap_mean_ci`, `leave_one_out_sensitivity` |

{zaksphysicslibrary-1.3.0 → zaksphysicslibrary-1.5.0}/PhysicsLibrary/__init__.py

@@ -55,4 +55,26 @@ from .models import (
     exponential_rise_model,
     gaussian_model,
     sinusoidal_model,
+)
+
+from .text_field_study import (
+    run_field_study_pipeline,
+    load_field_study_folder,
+    peek_fields,
+    flag_low_quality,
+    embed_text_fields,
+    compute_delta_vector,
+    compute_paired_similarity,
+    permutation_test_similarity,
+    wordcount_confound_check,
+)
+
+from .field_study_validation import (
+    run_validation_pipeline,
+    build_validation_summary,
+    cohens_d,
+    benjamini_hochberg,
+    wordcount_controlled_regression,
+    bootstrap_mean_ci,
+    leave_one_out_sensitivity,
 )

zaksphysicslibrary-1.5.0/PhysicsLibrary/field_study_validation.py

@@ -0,0 +1,433 @@
+"""
+field_study_validation.py
+----------------------------
+Statistical validation for the paired-similarity metric computed by
+text_field_study.py: for each field pair, is the same-subject
+similarity meaningfully higher than chance, how big is that effect, is
+it just a word-count artifact, how precise is the estimate, and how
+much does it depend on any one subject?
+
+run_validation_pipeline() is the single entry point (load -> embed ->
+compute similarity -> validate, self-contained so it can run as its own
+follow-up analysis without needing anything cached from a prior
+run_field_study_pipeline() call); the stage functions below are also
+exposed separately. Every function's docstring explains what the
+statistic actually means, for interpreting/writing up results — not
+just how to call it.
+"""
+
+import numpy as np
+import pandas as pd
+
+from .text_field_study import (
+    load_field_study_folder,
+    embed_text_fields,
+    compute_paired_similarity,
+    permutation_test_similarity,
+)
+
+
+def cohens_d(group1, group2):
+    """
+    Cohen's d: how far apart two distributions' means are, in units of
+    their pooled standard deviation. Unlike a p-value (which just says
+    "probably not the same"), this says how *big* the difference is —
+    the conventional rough guide is ~0.2 small, ~0.5 medium, ~0.8 large,
+    but those thresholds are a loose convention, not a hard rule.
+
+    Here: group1 is typically the same-subject similarity values (one
+    per subject), group2 the pooled null (chance-pairing) similarity
+    values — d tells you how far the "real" pairing sits from chance
+    pairing, in a way that doesn't depend on sample size the way a
+    p-value does.
+
+    Parameters
+    ----------
+    group1, group2 : array-like
+
+    Returns
+    -------
+    float
+    """
+    group1 = np.asarray(group1, dtype=float)
+    group2 = np.asarray(group2, dtype=float)
+    n1, n2 = len(group1), len(group2)
+    if n1 < 2 or n2 < 2:
+        # A standard deviation isn't defined from a single value — not
+        # enough subjects/null draws yet for this to mean anything.
+        return float("nan")
+    var1, var2 = group1.var(ddof=1), group2.var(ddof=1)
+    pooled_std = np.sqrt(((n1 - 1) * var1 + (n2 - 1) * var2) / (n1 + n2 - 2))
+    if pooled_std == 0:
+        return float("nan")
+    return float((group1.mean() - group2.mean()) / pooled_std)
+
+
+def benjamini_hochberg(p_values):
+    """
+    Benjamini-Hochberg false discovery rate correction. Running several
+    hypothesis tests (here: one permutation test per field pair) inflates
+    the chance that at least one comes back "significant" by luck alone —
+    this adjusts each p-value upward to control the *expected proportion*
+    of false positives among the tests you call significant, which is
+    less conservative (more power) than a stricter correction like
+    Bonferroni while still accounting for doing multiple tests.
+
+    The output is often called a "q-value": if you use q < 0.05 as your
+    cutoff, you're accepting that on average 5% of the pairs you call
+    significant across repeated studies like this one would be false
+    positives — not that any individual result has a 5% chance of being
+    wrong.
+
+    Parameters
+    ----------
+    p_values : array-like
+
+    Returns
+    -------
+    ndarray
+        Corrected p-values (q-values), same order as the input.
+    """
+    p_values = np.asarray(p_values, dtype=float)
+    n = len(p_values)
+    order = np.argsort(p_values)
+    ranked = p_values[order]
+    corrected = ranked * n / np.arange(1, n + 1)
+    # Enforce monotonicity (a lower-ranked p-value's correction can never
+    # end up smaller than a higher-ranked one's) via a reverse cumulative
+    # minimum — the standard BH step-up procedure.
+    corrected = np.minimum.accumulate(corrected[::-1])[::-1]
+    corrected = np.clip(corrected, 0, 1)
+    out = np.empty(n)
+    out[order] = corrected
+    return out
+
+
+def wordcount_controlled_regression(df, field_a, field_b, sim_col):
+    """
+    OLS regression: same-subject similarity ~ wordcount_<field_a> +
+    wordcount_<field_b>. Tests whether verbosity in either field predicts
+    similarity, controlling for the other. If a word-count coefficient
+    comes back significant (small p-value), that field's length is
+    predicting similarity independent of actual content overlap — a
+    "longer answers just look more similar" confound that would weaken a
+    claim based on the similarity metric alone. If neither is
+    significant, the observed similarity isn't just a length artifact.
+
+    Requires the `statsmodels` package.
+
+    Parameters
+    ----------
+    df : pandas.DataFrame
+        Must have wordcount_<field_a>, wordcount_<field_b>, and sim_col
+        columns (i.e. called on the output of run_field_study_pipeline,
+        or with those columns present some other way).
+    field_a, field_b : str
+    sim_col : str
+        The similarity column to use as the outcome, e.g. 'sim_<name>'.
+
+    Returns
+    -------
+    dict
+        r_squared : float
+            Proportion of variance in similarity explained by both word
+            counts together — small values mean word count barely
+            matters, which is the reassuring result for the confound
+            check.
+        coef_const, coef_wordcount_<field_a>, coef_wordcount_<field_b> : float
+            Regression coefficients (the intercept, and how much
+            similarity is predicted to change per additional word in
+            each field).
+        pvalue_const, pvalue_wordcount_<field_a>, pvalue_wordcount_<field_b> : float
+            Whether each coefficient is distinguishable from zero.
+        model : statsmodels RegressionResults
+            The full fitted model, for anyone who wants more than the
+            summary numbers above (e.g. confidence intervals, residual
+            diagnostics).
+    """
+    import statsmodels.api as sm
+
+    x = df[[f"wordcount_{field_a}", f"wordcount_{field_b}"]].astype(float)
+    x = sm.add_constant(x)
+    y = df[sim_col].astype(float)
+
+    # 3 parameters (intercept + 2 word-count covariates) need at least 3
+    # subjects to not be a singular/degenerate fit — with fewer, there's
+    # no unique answer, so return NaNs instead of letting statsmodels
+    # produce a nonsensical or all-NaN result silently.
+    if len(df) < x.shape[1]:
+        result = {"r_squared": float("nan"), "model": None}
+        for name in x.columns:
+            result[f"coef_{name}"] = float("nan")
+            result[f"pvalue_{name}"] = float("nan")
+        return result
+
+    model = sm.OLS(y, x).fit()
+
+    result = {"r_squared": float(model.rsquared), "model": model}
+    for name in x.columns:
+        result[f"coef_{name}"] = float(model.params[name])
+        result[f"pvalue_{name}"] = float(model.pvalues[name])
+    return result
+
+
+def bootstrap_mean_ci(values, n_boot=1000, ci=0.95, rng_seed=None):
+    """
+    Bootstrap confidence interval on the mean: resample the subjects
+    (with replacement, same sample size) n_boot times, take the mean
+    each time, and report the [lower, upper] percentiles of that
+    distribution of means. This doesn't assume the underlying similarity
+    values are normally distributed (unlike a textbook mean ± 1.96*SE
+    interval) — it just asks "how much would my estimate of the mean
+    wobble if I'd happened to sample a slightly different set of
+    subjects from the same population."
+
+    A narrow interval means the estimate is precise (more subjects, or
+    less subject-to-subject variability); a wide one means treat the
+    point estimate cautiously — you don't have enough subjects yet to
+    pin it down tightly.
+
+    Parameters
+    ----------
+    values : array-like
+        Per-subject values (e.g. same-subject similarity for one pair).
+    n_boot : int
+        Number of resamples.
+    ci : float
+        Confidence level, e.g. 0.95 for a 95% interval.
+    rng_seed : int or None
+
+    Returns
+    -------
+    dict
+        mean, ci_lower, ci_upper, boot_means (the full resampled-means
+        array, for anyone who wants to plot its distribution).
+    """
+    values = np.asarray(values, dtype=float)
+    rng = np.random.default_rng(rng_seed)
+    n = len(values)
+    boot_means = np.empty(n_boot)
+    for i in range(n_boot):
+        boot_means[i] = rng.choice(values, size=n, replace=True).mean()
+
+    alpha = 1 - ci
+    lower = np.percentile(boot_means, 100 * alpha / 2)
+    upper = np.percentile(boot_means, 100 * (1 - alpha / 2))
+    return {
+        "mean": float(values.mean()),
+        "ci_lower": float(lower),
+        "ci_upper": float(upper),
+        "boot_means": boot_means,
+    }
+
+
+def leave_one_out_sensitivity(values, ids=None):
+    """
+    Leave-one-out sensitivity check: recompute the mean with each subject
+    dropped, one at a time, and flag any subject whose removal shifts the
+    mean by more than one standard deviation of the resulting
+    (n_subjects-long) distribution of leave-one-out means.
+
+    This catches a single unusual subject quietly dominating your
+    result — e.g. one participant with an oddly high or low similarity
+    score dragging the whole-group mean toward it. A flagged subject
+    isn't necessarily wrong or worth excluding — it's worth looking at
+    that subject's actual data and deciding, not an automatic exclusion
+    rule.
+
+    Parameters
+    ----------
+    values : array-like
+        Per-subject values (e.g. same-subject similarity for one pair).
+    ids : array-like or None
+        Subject identifiers matching values' order, for readable output.
+        Defaults to positional indices if not given.
+
+    Returns
+    -------
+    dict
+        full_mean : float
+        loo_means : ndarray
+            The mean with each subject left out, in input order.
+        shifts : ndarray
+            full_mean minus each loo_means entry — how far removing that
+            subject moved the mean.
+        threshold : float
+            One standard deviation of loo_means — the cutoff used to flag.
+        flagged_ids : list
+            Subject ids whose |shift| exceeds threshold.
+    """
+    values = np.asarray(values, dtype=float)
+    n = len(values)
+    if ids is None:
+        ids = list(range(n))
+    else:
+        ids = list(ids)
+
+    full_mean = float(values.mean()) if n > 0 else float("nan")
+    if n < 2:
+        # Nothing to leave out-and-compare with fewer than 2 subjects —
+        # "does removing one subject change the mean" isn't meaningful
+        # when there's only one (or zero) to begin with.
+        return {
+            "full_mean": full_mean,
+            "loo_means": np.array([]),
+            "shifts": np.array([]),
+            "threshold": 0.0,
+            "flagged_ids": [],
+        }
+
+    loo_means = np.array([np.delete(values, i).mean() for i in range(n)])
+    shifts = full_mean - loo_means
+    threshold = float(loo_means.std(ddof=1))
+    flagged_mask = np.abs(shifts) > threshold
+
+    return {
+        "full_mean": full_mean,
+        "loo_means": loo_means,
+        "shifts": shifts,
+        "threshold": threshold,
+        "flagged_ids": [ids[i] for i in range(n) if flagged_mask[i]],
+    }
+
+
+def build_validation_summary(df, paired_fields, similarity_results, id_field="participant_id", n_boot=1000, rng_seed=None):
+    """
+    Folds every check above into one summary table, one row per field
+    pair — see each column's meaning below (and the stage functions'
+    own docstrings for the full explanation of each statistic).
+
+    Parameters
+    ----------
+    df : pandas.DataFrame
+        Needs wordcount_<field> columns for every field in paired_fields,
+        and id_field if you want subject ids (not positional indices) in
+        flagged_participant_ids.
+    paired_fields : list of (field_a, field_b, pair_name)
+    similarity_results : dict
+        As returned by compute_paired_similarity, computed with these
+        same paired_fields — reused rather than recomputed here, since
+        recomputing similarity means recomputing the (expensive)
+        embeddings.
+    id_field : str
+    n_boot : int
+    rng_seed : int or None
+
+    Returns
+    -------
+    pandas.DataFrame
+        One row per pair:
+        - pair, field_a, field_b : which fields this row covers
+        - observed_mean_similarity : mean same-subject similarity
+        - p_value : permutation-test p-value (is the observed mean
+          higher than chance-pairing, one-tailed)
+        - p_value_fdr : p_value after Benjamini-Hochberg correction
+          across all pairs in this table — use this one, not the raw
+          p_value, once you're looking at more than one pair
+        - cohens_d : effect size (same-subject similarity vs. the pooled
+          chance-pairing null), pooled-std units
+        - wc_coef_a/wc_pvalue_a, wc_coef_b/wc_pvalue_b : word-count
+          regression coefficient/p-value for field_a's and field_b's
+          word count respectively, controlling for each other
+          (wordcount_controlled_regression) — a significant coefficient
+          here means part of the similarity effect may just be verbosity
+        - regression_r_squared : how much of the similarity variance
+          both word counts together explain (low = reassuring)
+        - ci_lower, ci_upper : bootstrap 95% CI on observed_mean_similarity
+        - n_flagged_loo : how many subjects were flagged by the
+          leave-one-out check (0 is the reassuring result)
+        - flagged_participant_ids : which ones, comma-separated
+    """
+    rows = []
+    p_values = []
+    for field_a, field_b, name in paired_fields:
+        result = similarity_results[name]
+        same = result["same_subject_similarity"]
+        null_values = result["null_values"]
+        null_shuffle_means = result["null_shuffle_means"]
+
+        perm = permutation_test_similarity(same, null_shuffle_means)
+        d = cohens_d(same, null_values)
+        reg = wordcount_controlled_regression(df, field_a, field_b, f"sim_{name}")
+        ci = bootstrap_mean_ci(same, n_boot=n_boot, rng_seed=rng_seed)
+        ids = df[id_field].tolist() if id_field in df.columns else None
+        loo = leave_one_out_sensitivity(same, ids=ids)
+
+        rows.append({
+            "pair": name,
+            "field_a": field_a,
+            "field_b": field_b,
+            "observed_mean_similarity": perm["observed_mean_similarity"],
+            "p_value": perm["p_value"],
+            "cohens_d": d,
+            "wc_coef_a": reg[f"coef_wordcount_{field_a}"],
+            "wc_pvalue_a": reg[f"pvalue_wordcount_{field_a}"],
+            "wc_coef_b": reg[f"coef_wordcount_{field_b}"],
+            "wc_pvalue_b": reg[f"pvalue_wordcount_{field_b}"],
+            "regression_r_squared": reg["r_squared"],
+            "ci_lower": ci["ci_lower"],
+            "ci_upper": ci["ci_upper"],
+            "n_flagged_loo": len(loo["flagged_ids"]),
+            "flagged_participant_ids": ",".join(str(x) for x in loo["flagged_ids"]),
+        })
+        p_values.append(perm["p_value"])
+
+    p_fdr = benjamini_hochberg(np.array(p_values))
+    for row, q in zip(rows, p_fdr):
+        row["p_value_fdr"] = float(q)
+
+    return pd.DataFrame(rows)
+
+
+def run_validation_pipeline(folder_path, text_fields, paired_fields,
+                             model_name="all-MiniLM-L6-v2", n_null=200,
+                             n_boot=1000, rng_seed=None, file_glob="P-*.json",
+                             id_field="participant_id"):
+    """
+    Full validation pipeline: load -> embed -> compute paired similarity
+    -> build the validation summary table (see build_validation_summary
+    for what each column means). Self-contained — recomputes embeddings
+    rather than reusing a prior run_field_study_pipeline() call, so this
+    can run as its own follow-up analysis.
+
+    Parameters
+    ----------
+    folder_path : str
+    text_fields : list of str
+        Same meaning as in run_field_study_pipeline.
+    paired_fields : list of (field_a, field_b, pair_name)
+        Which fields to compare. Required here (unlike
+        run_field_study_pipeline, where it's optional) — there's nothing
+        to validate without at least one pair.
+    model_name : str
+    n_null : int
+        Permutation shuffles for the null distribution.
+    n_boot : int
+        Bootstrap resamples for the confidence interval.
+    rng_seed : int or None
+    file_glob : str
+    id_field : str
+        Column to use as subject identifiers in the leave-one-out flags.
+
+    Returns
+    -------
+    pandas.DataFrame
+        See build_validation_summary.
+    """
+    if not paired_fields:
+        raise ValueError("paired_fields is required — nothing to validate without at least one pair.")
+
+    df = load_field_study_folder(folder_path, text_fields, file_glob=file_glob)
+    fields_to_embed = sorted({f for field_a, field_b, _ in paired_fields for f in (field_a, field_b)})
+    embeddings = embed_text_fields(df, fields_to_embed, model_name=model_name)
+    similarity_results = compute_paired_similarity(embeddings, paired_fields, n_null=n_null, rng_seed=rng_seed)
+
+    # build_validation_summary's regression step needs sim_<name> columns
+    # on df, same as run_field_study_pipeline adds them.
+    for field_a, field_b, name in paired_fields:
+        df[f"sim_{name}"] = similarity_results[name]["same_subject_similarity"]
+
+    return build_validation_summary(
+        df, paired_fields, similarity_results,
+        id_field=id_field, n_boot=n_boot, rng_seed=rng_seed,
+    )

{zaksphysicslibrary-1.3.0 → zaksphysicslibrary-1.5.0}/PhysicsLibrary/processing_TDT.py

@@ -278,10 +278,19 @@ def get_event_markers(data):
     toggle each store's markers independently instead of dumping every
     store onto the plot at once (which overlaps into an unreadable mess).
 
+    Every epoc is a state that goes high (onset) and later low (offset) —
+    for a lever press that's press/release, for a pump or light that's
+    on/off. Both are emitted here, each tagged with a 'phase' key ('high'
+    for onset, 'low' for offset), so a caller can decide per store whether
+    it cares about both edges (e.g. how long the pump ran) or only one
+    (e.g. just the press, not the release) instead of that decision being
+    baked into the library.
+
     The 'Note' store keeps its original behaviour: each onset becomes a
-    marker labeled with the actual note text (from Notes.txt), since
-    that's the one store meant for free-text annotations. Every other
-    epoc store gets one marker per onset event, labeled with the store
+    marker labeled with the actual note text (from Notes.txt), and has no
+    offset/'low' marker — free-text annotations are instantaneous, not a
+    state with a duration. Every other epoc store gets one 'high' marker
+    per onset and one 'low' marker per offset, both labeled with the store
     name.
 
     Nothing about which stores exist is hardcoded — this walks whatever
@@ -294,7 +303,8 @@ def get_event_markers(data):
         - time  : float
         - label : str
         - color : str
-        - store : str  (source epoc store name, for grouping/toggling)
+        - store : str   (source epoc store name, for grouping/toggling)
+        - phase : str   ('high'=onset, 'low'=offset; omitted for Note markers)
     """
     if not hasattr(data, 'epocs'):
         return []
@@ -336,6 +346,20 @@ def get_event_markers(data):
                     'label': display_name,
                     'color': color,
                     'store': display_name,
+                    'phase': 'high',
+                })
+            offsets = getattr(epoc, 'offset', [])
+            for t in offsets:
+                # TDT represents a strobe still active at recording end with
+                # offset=inf — not a real timestamp, nothing to plot.
+                if not np.isfinite(t):
+                    continue
+                markers.append({
+                    'time':  float(t),
+                    'label': display_name,
+                    'color': color,
+                    'store': display_name,
+                    'phase': 'low',
                 })
 
     return markers

zaksphysicslibrary-1.5.0/PhysicsLibrary/text_field_study.py

@@ -0,0 +1,405 @@
+"""
+text_field_study.py
+---------------------
+Generic pipeline for a study where each subject produces one JSON file
+with several free-text fields, and you want to compare specific pairs
+of fields to each other (e.g. "does this answer track that one for the
+same subject"). Domain-agnostic by design — field names and which pairs
+to compare are supplied by the caller via arguments, not hardcoded
+here, so this works for any text-field study, not one specific one.
+
+run_field_study_pipeline() is the single entry point for running the
+whole thing at once; the stage functions below are also exposed for
+anyone who wants an intermediate result (embeddings, raw null
+distributions) directly.
+"""
+
+import glob
+import json
+import os
+
+import numpy as np
+import pandas as pd
+from scipy import stats
+
+
+def peek_fields(folder_path, file_glob="P-*.json"):
+    """
+    Returns the field names (dict keys) found in the first file matching
+    file_glob in folder_path, without loading the whole folder — lets a
+    caller (e.g. a GUI) show a user their actual field names to assign
+    to groups, rather than requiring them to already know/type them.
+
+    Parameters
+    ----------
+    folder_path : str
+    file_glob : str
+
+    Returns
+    -------
+    list of str
+    """
+    paths = sorted(glob.glob(os.path.join(folder_path, file_glob)))
+    if not paths:
+        raise ValueError(f"No files matching '{file_glob}' found in {folder_path}")
+    with open(paths[0], "r", encoding="utf-8") as fh:
+        return list(json.load(fh).keys())
+
+
+def load_field_study_folder(folder_path, text_fields, file_glob="P-*.json"):
+    """
+    Load every file matching file_glob in folder_path into a DataFrame
+    (one row per subject) and add a wordcount_<field> column for each
+    text field — a covariate for later analysis, not itself a feature.
+
+    Parameters
+    ----------
+    folder_path : str
+    text_fields : list of str
+        Every free-text field to word-count.
+    file_glob : str
+        Filename pattern within folder_path, e.g. "P-*.json".
+
+    Returns
+    -------
+    pandas.DataFrame
+    """
+    paths = sorted(glob.glob(os.path.join(folder_path, file_glob)))
+    if not paths:
+        raise ValueError(f"No files matching '{file_glob}' found in {folder_path}")
+
+    records = []
+    for path in paths:
+        with open(path, "r", encoding="utf-8") as fh:
+            records.append(json.load(fh))
+    df = pd.DataFrame(records)
+
+    for field in text_fields:
+        text = df[field].fillna("") if field in df.columns else ""
+        df[f"wordcount_{field}"] = text.apply(lambda s: len(str(s).split()))
+
+    return df
+
+
+def flag_low_quality(df, text_fields, min_words=5):
+    """
+    Flags (doesn't drop) rows with near-empty responses — noise, not
+    signal, but the caller decides whether/how to exclude them, so
+    nothing here is silently discarded.
+
+    Adds a low_quality_<field> bool column per text field (True if that
+    field's word count is below min_words) and an any_low_quality column
+    (True if any field is flagged for that subject). Must be called
+    after word counts exist (i.e. after load_field_study_folder).
+
+    Parameters
+    ----------
+    df : pandas.DataFrame
+        As returned by load_field_study_folder — must have
+        wordcount_<field> columns for text_fields.
+    text_fields : list of str
+    min_words : int
+        Fields with fewer words than this are flagged.
+
+    Returns
+    -------
+    pandas.DataFrame
+        The same df, with the new columns added.
+    """
+    flag_cols = []
+    for field in text_fields:
+        col = f"low_quality_{field}"
+        df[col] = df[f"wordcount_{field}"] < min_words
+        flag_cols.append(col)
+    df["any_low_quality"] = df[flag_cols].any(axis=1)
+    return df
+
+
+def embed_text_fields(df, fields, model_name="all-MiniLM-L6-v2"):
+    """
+    Embed each of `fields` with a sentence-transformers model,
+    L2-normalized (so a plain dot product below is the cosine similarity).
+    Flat by field name — nothing about which fields get compared to
+    which needs to be decided at embedding time, only when you call
+    compute_delta_vector/compute_paired_similarity afterward.
+
+    Parameters
+    ----------
+    df : pandas.DataFrame
+        As returned by load_field_study_folder.
+    fields : list of str
+        Which fields to embed (typically every field referenced by your
+        paired_fields/delta_pair — no need to embed fields you're not
+        actually comparing).
+    model_name : str
+
+    Returns
+    -------
+    dict
+        {field: (n_subjects, dim) ndarray, ...}
+    """
+    from sentence_transformers import SentenceTransformer
+
+    model = SentenceTransformer(model_name)
+    embeddings = {}
+    for field in fields:
+        texts = df[field].fillna("").astype(str).tolist()
+        embeddings[field] = np.asarray(model.encode(texts, normalize_embeddings=True))
+    return embeddings
+
+
+def compute_delta_vector(embeddings, field_from, field_to):
+    """
+    delta = vec(field_to) - vec(field_from), per subject, plus its
+    magnitude — e.g. how much a subject's answer changed between two
+    related fields.
+
+    Parameters
+    ----------
+    embeddings : dict
+        As returned by embed_text_fields.
+    field_from, field_to : str
+
+    Returns
+    -------
+    (delta_vectors, delta_magnitude) : (ndarray, ndarray)
+    """
+    vectors = embeddings[field_to] - embeddings[field_from]
+    magnitude = np.linalg.norm(vectors, axis=1)
+    return vectors, magnitude
+
+
+def compute_paired_similarity(embeddings, paired_fields, n_null=200, rng_seed=None):
+    """
+    Cosine similarity between each pair of fields for the same subject,
+    plus a null distribution built by repeatedly shuffling one field's
+    vectors across subjects and recomputing similarity — the basis for a
+    later permutation test on whether same-subject similarity is higher
+    than chance pairing.
+
+    Each shuffle is a full permutation across all subjects at once; all
+    n_null * n_subjects values across every shuffle are pooled into one
+    null distribution per pair (not per subject — "chance pairing" is a
+    property of the pairing process, not of any one subject). A
+    permutation occasionally maps a subject to themselves (identity
+    mapping isn't excluded); for a reasonable sample size this is a
+    negligible bias, not worth the extra complexity of a derangement.
+
+    Parameters
+    ----------
+    embeddings : dict
+        As returned by embed_text_fields.
+    paired_fields : list of (field_a, field_b, pair_name)
+        Which two fields to compare, and a name for that comparison.
+    n_null : int
+        Number of random shuffles to build the null distribution from.
+    rng_seed : int or None
+
+    Returns
+    -------
+    dict
+        {pair_name: {
+            'same_subject_similarity': (n_subjects,) ndarray,
+            'null_mean': float,
+            'null_std': float,
+            'null_values': (n_null * n_subjects,) ndarray,
+            'null_shuffle_means': (n_null,) ndarray,
+                the mean similarity within each individual shuffle —
+                this (not null_values) is the actual null distribution
+                for a permutation test of the *mean* same-subject
+                similarity; see permutation_test_similarity.
+        }, ...}
+    """
+    rng = np.random.default_rng(rng_seed)
+    results = {}
+    for field_a, field_b, name in paired_fields:
+        vecs_a = embeddings[field_a]
+        vecs_b = embeddings[field_b]
+        n = vecs_a.shape[0]
+
+        same_subject_similarity = np.sum(vecs_a * vecs_b, axis=1)
+
+        null_values = np.empty(n_null * n)
+        null_shuffle_means = np.empty(n_null)
+        for i in range(n_null):
+            shuffled = vecs_b[rng.permutation(n)]
+            shuffle_sims = np.sum(vecs_a * shuffled, axis=1)
+            null_values[i * n:(i + 1) * n] = shuffle_sims
+            null_shuffle_means[i] = shuffle_sims.mean()
+
+        results[name] = {
+            "same_subject_similarity": same_subject_similarity,
+            "null_mean": float(null_values.mean()),
+            "null_std": float(null_values.std()),
+            "null_values": null_values,
+            "null_shuffle_means": null_shuffle_means,
+        }
+    return results
+
+
+def permutation_test_similarity(same_subject_similarity, null_shuffle_means):
+    """
+    Permutation test: is the observed mean same-subject similarity
+    higher than expected under random pairing? Compares the observed
+    mean against the distribution of per-shuffle means (not the pooled
+    individual null_values — the test statistic is the mean, so its
+    null distribution has to be built from the same statistic computed
+    under each shuffle, not from individual chance-pairing similarities).
+
+    One-tailed (is same-subject similarity higher than chance, not just
+    different) — add-one smoothing avoids a p-value of exactly 0 for a
+    finite number of shuffles.
+
+    Parameters
+    ----------
+    same_subject_similarity : ndarray
+        Per-subject observed similarity (one pair's worth).
+    null_shuffle_means : ndarray
+        Per-shuffle mean similarity, as returned by
+        compute_paired_similarity.
+
+    Returns
+    -------
+    dict
+        observed_mean_similarity, p_value,
+        effect_size (observed mean minus the null shuffle-mean
+        distribution's own mean, in that distribution's std units),
+        null_shuffle_mean, null_shuffle_std.
+    """
+    observed = float(np.mean(same_subject_similarity))
+    n_null = len(null_shuffle_means)
+    p_value = (np.sum(null_shuffle_means >= observed) + 1) / (n_null + 1)
+
+    shuffle_mean = float(null_shuffle_means.mean())
+    shuffle_std = float(null_shuffle_means.std())
+    effect_size = (observed - shuffle_mean) / shuffle_std if shuffle_std > 0 else float("nan")
+
+    return {
+        "observed_mean_similarity": observed,
+        "p_value": float(p_value),
+        "effect_size": effect_size,
+        "null_shuffle_mean": shuffle_mean,
+        "null_shuffle_std": shuffle_std,
+    }
+
+
+def wordcount_confound_check(df, paired_fields):
+    """
+    Checks whether word count predicts similarity — a "more words = more
+    similar" confound would show up as a significant positive
+    correlation here, which would undercut any claim based on the
+    similarity metric alone.
+
+    Pearson correlation between each pair's combined word count
+    (wordcount_<field_a> + wordcount_<field_b>) and its similarity
+    column (sim_<name>) across subjects.
+
+    Parameters
+    ----------
+    df : pandas.DataFrame
+        Must have wordcount_<field_a>, wordcount_<field_b>, and
+        sim_<name> columns for each pair (i.e. called after
+        run_field_study_pipeline's word-count and similarity steps).
+    paired_fields : list of (field_a, field_b, pair_name)
+
+    Returns
+    -------
+    dict
+        {pair_name: {'r': float, 'p_value': float}, ...}
+        Both are NaN if there are fewer than 2 subjects, or every
+        subject has the same word count — a correlation genuinely isn't
+        defined in either case, not something to error out over.
+    """
+    results = {}
+    for field_a, field_b, name in paired_fields:
+        wordcount = df[f"wordcount_{field_a}"] + df[f"wordcount_{field_b}"]
+        similarity = df[f"sim_{name}"]
+        if len(wordcount) < 2 or wordcount.nunique() < 2 or similarity.nunique() < 2:
+            results[name] = {"r": float("nan"), "p_value": float("nan")}
+            continue
+        r, p_value = stats.pearsonr(wordcount, similarity)
+        results[name] = {"r": float(r), "p_value": float(p_value)}
+    return results
+
+
+def run_field_study_pipeline(folder_path, text_fields, delta_pair=None, paired_fields=None,
+                              model_name="all-MiniLM-L6-v2", n_null=200, rng_seed=None,
+                              file_glob="P-*.json", min_words=5):
+    """
+    Full pipeline: load -> flag low-quality responses -> embed -> optional
+    delta vector -> optional paired similarity (+ permutation test +
+    word-count confound check), folded into one DataFrame (one row per
+    subject). This is the single entry point for running the whole thing
+    at once (e.g. as one background job); the stage functions above are
+    available separately for anyone who wants an intermediate result
+    (e.g. the raw null distributions, not carried into the DataFrame
+    here — only their mean/std/permutation-test summary are).
+
+    Parameters
+    ----------
+    folder_path : str
+    text_fields : list of str
+        Every free-text field to load/word-count.
+    delta_pair : (field_from, field_to) or None
+        If given, adds a delta_magnitude column:
+        ||vec(field_to) - vec(field_from)||.
+    paired_fields : list of (field_a, field_b, pair_name) or None
+        If given, adds per pair: sim_<name>, null_mean_<name>,
+        null_std_<name> (individual chance-pairing similarity stats),
+        pvalue_<name>/effect_size_<name> (permutation test of whether
+        mean same-subject similarity beats chance pairing), and
+        wc_confound_r_<name>/wc_confound_p_<name> (does combined word
+        count predict similarity — a possible "more words = more
+        similar" confound). The test-level values (p-value, effect
+        size, confound r) are the same for every row, same as
+        null_mean_/null_std_ already were — one number per pair, not
+        per subject.
+    model_name : str
+    n_null : int
+    rng_seed : int or None
+    file_glob : str
+    min_words : int
+        Fields with fewer words than this get flagged (not dropped) —
+        see flag_low_quality.
+
+    Returns
+    -------
+    pandas.DataFrame
+    """
+    df = load_field_study_folder(folder_path, text_fields, file_glob=file_glob)
+    df = flag_low_quality(df, text_fields, min_words=min_words)
+
+    # Only embed fields actually referenced by delta_pair/paired_fields —
+    # embedding is the expensive step, no reason to run it over fields
+    # nobody's comparing.
+    fields_to_embed = set()
+    if delta_pair is not None:
+        fields_to_embed.update(delta_pair)
+    for field_a, field_b, _ in (paired_fields or []):
+        fields_to_embed.update([field_a, field_b])
+    embeddings = embed_text_fields(df, sorted(fields_to_embed), model_name=model_name)
+
+    if delta_pair is not None:
+        field_from, field_to = delta_pair
+        _, magnitude = compute_delta_vector(embeddings, field_from, field_to)
+        df["delta_magnitude"] = magnitude
+
+    if paired_fields:
+        results = compute_paired_similarity(embeddings, paired_fields, n_null=n_null, rng_seed=rng_seed)
+        for name, result in results.items():
+            df[f"sim_{name}"] = result["same_subject_similarity"]
+            df[f"null_mean_{name}"] = result["null_mean"]
+            df[f"null_std_{name}"] = result["null_std"]
+
+            test = permutation_test_similarity(
+                result["same_subject_similarity"], result["null_shuffle_means"]
+            )
+            df[f"pvalue_{name}"] = test["p_value"]
+            df[f"effect_size_{name}"] = test["effect_size"]
+
+        confound = wordcount_confound_check(df, paired_fields)
+        for name, result in confound.items():
+            df[f"wc_confound_r_{name}"] = result["r"]
+            df[f"wc_confound_p_{name}"] = result["p_value"]
+
+    return df

{zaksphysicslibrary-1.3.0 → zaksphysicslibrary-1.5.0}/README.md

@@ -13,6 +13,8 @@ Data processing and analysis library backing [PhysicsAnalysis](https://github.co
   - Generic **Excel / CSV / TSV / plain text**, with automatic sub-table detection for side-by-side data layouts on one sheet
 - **Processes signals** — bleach correction, denoising, Z-score PETH slicing, FFT with peak annotation, slope/segment analysis
 - **Fits curves** — linear, single/double exponential, exponential rise, Gaussian, sinusoidal, and a photon-entanglement visibility model, all via `scipy.optimize.curve_fit`
+- **Analyses text field studies** — one JSON file per subject with several free-text fields; pick any pair of fields to compare directly (e.g. does the answer to one question track another for the same subject). Word counts, data-quality flagging, sentence-transformers embeddings, an optional delta-vector magnitude between two fields, and an optional paired-similarity metric per pair with a permutation test and a word-count confound check. Domain-agnostic — field names and which pairs to compare are supplied by the caller, nothing is hardcoded to one study
+- **Validates the similarity metric statistically** — Benjamini-Hochberg FDR-corrected permutation-test p-values, Cohen's d effect size, a word-count-controlled OLS regression (statsmodels), a bootstrap confidence interval on the mean, and a leave-one-out sensitivity check, one row per field pair, with docstrings explaining what each statistic means
 
 ---
 
@@ -27,6 +29,8 @@ PhysicsLibrary/
   processing_TDT.py          TDT tank reading, bleach correction, denoising, event markers
   analysis.py                 PETH/Z-score, FFT, slope segments, curve-fit runner
   models.py                    Parametric model functions for curve fitting
+  text_field_study.py           Grouped-text-field study pipeline (embeddings, delta vector, paired similarity)
+  field_study_validation.py      Statistical validation for the paired-similarity metric (permutation test, Cohen's d, regression, bootstrap CI, leave-one-out)
   loaders/
     tdt_loader.py               Wraps processing_TDT into a Dataset
     oxysoft_loader.py            Oxysoft .txt parsing (folder + single-file) into a Dataset
@@ -52,7 +56,8 @@ git+https://github.com/zakgm2/PhysicsLibrary.git
 ### Requirements
 
 - Python 3.10+
-- `numpy`, `scipy`, `tdt` (installed automatically)
+- `numpy`, `scipy`, `tdt`, `pandas`, `sentence-transformers`, `statsmodels` (installed automatically)
+- `sentence-transformers` pulls in `torch`/`transformers` as transitive dependencies — a genuinely heavy install (hundreds of MB) if you only need the signal-processing side; only actually loaded when you call `embed_text_fields`/`run_field_study_pipeline`
 - `openpyxl` — only needed for `.xlsx`/`.xls` files; imported lazily with a clear error if missing when you actually try to load Excel
 
 ---
@@ -100,6 +105,34 @@ result = pl.fit_model_to_segment(x_seg, y_seg, pl.single_exponential_model, p0_f
 result["popt"], result["r2"], result["y_fit"]
 ```
 
+```python
+# Text field study — one JSON file per subject, e.g. P-0001.json. Pick
+# pairs of fields to compare directly; no grouping concept needed.
+fields = pl.peek_fields(folder_path)              # see what fields exist before picking pairs
+df = pl.run_field_study_pipeline(
+    folder_path,
+    text_fields=["q1", "q2", "q3", "q4"],
+    delta_pair=("q1", "q2"),                        # optional: how much did q2 change from q1
+    paired_fields=[("q1", "q3", "pair1")],           # optional: does q1 track q3
+)
+# df has one row per subject: wordcount_<field>, low_quality_<field>, delta_magnitude,
+# sim_<pair>, pvalue_<pair>, effect_size_<pair>, wc_confound_r_<pair>, ...
+```
+
+```python
+# Statistical validation of the paired-similarity metric — one row per pair
+summary = pl.run_validation_pipeline(
+    folder_path,
+    text_fields=["q1", "q3"],
+    paired_fields=[("q1", "q3", "pair1")],
+)
+# summary: p_value, p_value_fdr, cohens_d, wc_coef_a/b + wc_pvalue_a/b,
+# regression_r_squared, ci_lower/ci_upper, n_flagged_loo, flagged_participant_ids
+```
+
+See [FIELD_STUDY_METHODOLOGY.md](FIELD_STUDY_METHODOLOGY.md) for why each statistic in the
+validation step is a sound, standard technique — useful if anyone asks.
+
 See [CHANGELOG.md](CHANGELOG.md) for the version history.
 
 ---
@@ -110,8 +143,10 @@ Everything importable from `PhysicsLibrary` directly:
 
 | Category | Names |
 |----------|-------|
-| Format detection | `choose_file`, `detect_format`, `detect_format_file`, `DataFormat`, `Dataset` |
+| Format detection | `detect_format`, `detect_format_file`, `DataFormat`, `Dataset` |
 | Loading | `load_dataset`, `load_dataset_file`, `load_any_file`, `load_pt2` |
 | TDT processing | `process_tdt_folder`, `validate_tdt_folder`, `get_tdt_struct`, `get_plot_data`, `correct_bleaching`, `denoise_signal`, `get_event_markers` |
 | Analysis | `get_zscore_slice`, `smooth_signal`, `bin_for_heatmap`, `compute_fft_slice`, `annotate_fft_peaks`, `compute_slope_segment`, `fit_model_to_segment` |
 | Curve fit models | `linear_model`, `single_exponential_model`, `exponential_rise_model`, `double_exponential_model`, `gaussian_model`, `sinusoidal_model`, `visibility_model` |
+| Text field study | `run_field_study_pipeline`, `load_field_study_folder`, `peek_fields`, `flag_low_quality`, `embed_text_fields`, `compute_delta_vector`, `compute_paired_similarity`, `permutation_test_similarity`, `wordcount_confound_check` |
+| Field study validation | `run_validation_pipeline`, `build_validation_summary`, `cohens_d`, `benjamini_hochberg`, `wordcount_controlled_regression`, `bootstrap_mean_ci`, `leave_one_out_sensitivity` |

{zaksphysicslibrary-1.3.0 → zaksphysicslibrary-1.5.0/ZaksPhysicsLibrary.egg-info}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: ZaksPhysicsLibrary
-Version: 1.3.0
-Summary: Data processing and analysis library for TDT, Oxysoft NIRS, and Terranova EFNMR lab data
+Version: 1.5.0
+Summary: Data processing and analysis library for TDT, Oxysoft NIRS, Terranova EFNMR lab data, and grouped-text-field studies
 Author: zakgm2
 License-Expression: MIT
 Project-URL: Homepage, https://github.com/zakgm2/PhysicsLibrary
 Project-URL: Repository, https://github.com/zakgm2/PhysicsLibrary
 Project-URL: Changelog, https://github.com/zakgm2/PhysicsLibrary/blob/main/CHANGELOG.md
-Keywords: physics,neuroscience,fibre-photometry,NIRS,TDT,EFNMR,signal-processing
+Keywords: physics,neuroscience,fibre-photometry,NIRS,TDT,EFNMR,signal-processing,text-analysis,nlp
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -21,6 +21,9 @@ License-File: LICENSE
 Requires-Dist: numpy
 Requires-Dist: scipy
 Requires-Dist: tdt
+Requires-Dist: pandas
+Requires-Dist: sentence-transformers
+Requires-Dist: statsmodels
 Dynamic: license-file
 
 # PhysicsLibrary
@@ -38,6 +41,8 @@ Data processing and analysis library backing [PhysicsAnalysis](https://github.co
   - Generic **Excel / CSV / TSV / plain text**, with automatic sub-table detection for side-by-side data layouts on one sheet
 - **Processes signals** — bleach correction, denoising, Z-score PETH slicing, FFT with peak annotation, slope/segment analysis
 - **Fits curves** — linear, single/double exponential, exponential rise, Gaussian, sinusoidal, and a photon-entanglement visibility model, all via `scipy.optimize.curve_fit`
+- **Analyses text field studies** — one JSON file per subject with several free-text fields; pick any pair of fields to compare directly (e.g. does the answer to one question track another for the same subject). Word counts, data-quality flagging, sentence-transformers embeddings, an optional delta-vector magnitude between two fields, and an optional paired-similarity metric per pair with a permutation test and a word-count confound check. Domain-agnostic — field names and which pairs to compare are supplied by the caller, nothing is hardcoded to one study
+- **Validates the similarity metric statistically** — Benjamini-Hochberg FDR-corrected permutation-test p-values, Cohen's d effect size, a word-count-controlled OLS regression (statsmodels), a bootstrap confidence interval on the mean, and a leave-one-out sensitivity check, one row per field pair, with docstrings explaining what each statistic means
 
 ---
 
@@ -52,6 +57,8 @@ PhysicsLibrary/
   processing_TDT.py          TDT tank reading, bleach correction, denoising, event markers
   analysis.py                 PETH/Z-score, FFT, slope segments, curve-fit runner
   models.py                    Parametric model functions for curve fitting
+  text_field_study.py           Grouped-text-field study pipeline (embeddings, delta vector, paired similarity)
+  field_study_validation.py      Statistical validation for the paired-similarity metric (permutation test, Cohen's d, regression, bootstrap CI, leave-one-out)
   loaders/
     tdt_loader.py               Wraps processing_TDT into a Dataset
     oxysoft_loader.py            Oxysoft .txt parsing (folder + single-file) into a Dataset
@@ -77,7 +84,8 @@ git+https://github.com/zakgm2/PhysicsLibrary.git
 ### Requirements
 
 - Python 3.10+
-- `numpy`, `scipy`, `tdt` (installed automatically)
+- `numpy`, `scipy`, `tdt`, `pandas`, `sentence-transformers`, `statsmodels` (installed automatically)
+- `sentence-transformers` pulls in `torch`/`transformers` as transitive dependencies — a genuinely heavy install (hundreds of MB) if you only need the signal-processing side; only actually loaded when you call `embed_text_fields`/`run_field_study_pipeline`
 - `openpyxl` — only needed for `.xlsx`/`.xls` files; imported lazily with a clear error if missing when you actually try to load Excel
 
 ---
@@ -125,6 +133,34 @@ result = pl.fit_model_to_segment(x_seg, y_seg, pl.single_exponential_model, p0_f
 result["popt"], result["r2"], result["y_fit"]
 ```
 
+```python
+# Text field study — one JSON file per subject, e.g. P-0001.json. Pick
+# pairs of fields to compare directly; no grouping concept needed.
+fields = pl.peek_fields(folder_path)              # see what fields exist before picking pairs
+df = pl.run_field_study_pipeline(
+    folder_path,
+    text_fields=["q1", "q2", "q3", "q4"],
+    delta_pair=("q1", "q2"),                        # optional: how much did q2 change from q1
+    paired_fields=[("q1", "q3", "pair1")],           # optional: does q1 track q3
+)
+# df has one row per subject: wordcount_<field>, low_quality_<field>, delta_magnitude,
+# sim_<pair>, pvalue_<pair>, effect_size_<pair>, wc_confound_r_<pair>, ...
+```
+
+```python
+# Statistical validation of the paired-similarity metric — one row per pair
+summary = pl.run_validation_pipeline(
+    folder_path,
+    text_fields=["q1", "q3"],
+    paired_fields=[("q1", "q3", "pair1")],
+)
+# summary: p_value, p_value_fdr, cohens_d, wc_coef_a/b + wc_pvalue_a/b,
+# regression_r_squared, ci_lower/ci_upper, n_flagged_loo, flagged_participant_ids
+```
+
+See [FIELD_STUDY_METHODOLOGY.md](FIELD_STUDY_METHODOLOGY.md) for why each statistic in the
+validation step is a sound, standard technique — useful if anyone asks.
+
 See [CHANGELOG.md](CHANGELOG.md) for the version history.
 
 ---
@@ -135,8 +171,10 @@ Everything importable from `PhysicsLibrary` directly:
 
 | Category | Names |
 |----------|-------|
-| Format detection | `choose_file`, `detect_format`, `detect_format_file`, `DataFormat`, `Dataset` |
+| Format detection | `detect_format`, `detect_format_file`, `DataFormat`, `Dataset` |
 | Loading | `load_dataset`, `load_dataset_file`, `load_any_file`, `load_pt2` |
 | TDT processing | `process_tdt_folder`, `validate_tdt_folder`, `get_tdt_struct`, `get_plot_data`, `correct_bleaching`, `denoise_signal`, `get_event_markers` |
 | Analysis | `get_zscore_slice`, `smooth_signal`, `bin_for_heatmap`, `compute_fft_slice`, `annotate_fft_peaks`, `compute_slope_segment`, `fit_model_to_segment` |
 | Curve fit models | `linear_model`, `single_exponential_model`, `exponential_rise_model`, `double_exponential_model`, `gaussian_model`, `sinusoidal_model`, `visibility_model` |
+| Text field study | `run_field_study_pipeline`, `load_field_study_folder`, `peek_fields`, `flag_low_quality`, `embed_text_fields`, `compute_delta_vector`, `compute_paired_similarity`, `permutation_test_similarity`, `wordcount_confound_check` |
+| Field study validation | `run_validation_pipeline`, `build_validation_summary`, `cohens_d`, `benjamini_hochberg`, `wordcount_controlled_regression`, `bootstrap_mean_ci`, `leave_one_out_sensitivity` |

{zaksphysicslibrary-1.3.0 → zaksphysicslibrary-1.5.0}/ZaksPhysicsLibrary.egg-info/SOURCES.txt

@@ -4,10 +4,12 @@ pyproject.toml
 PhysicsLibrary/__init__.py
 PhysicsLibrary/analysis.py
 PhysicsLibrary/dataset.py
+PhysicsLibrary/field_study_validation.py
 PhysicsLibrary/file_parser.py
 PhysicsLibrary/file_parser_generic.py
 PhysicsLibrary/models.py
 PhysicsLibrary/processing_TDT.py
+PhysicsLibrary/text_field_study.py
 PhysicsLibrary/loaders/__init__.py
 PhysicsLibrary/loaders/oxysoft_loader.py
 PhysicsLibrary/loaders/pt2_loader.py

zaksphysicslibrary-1.5.0/ZaksPhysicsLibrary.egg-info/requires.txt

@@ -0,0 +1,6 @@
+numpy
+scipy
+tdt
+pandas
+sentence-transformers
+statsmodels

{zaksphysicslibrary-1.3.0 → zaksphysicslibrary-1.5.0}/pyproject.toml

@@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ZaksPhysicsLibrary"
-version = "1.3.0"
-description = "Data processing and analysis library for TDT, Oxysoft NIRS, and Terranova EFNMR lab data"
+version = "1.5.0"
+description = "Data processing and analysis library for TDT, Oxysoft NIRS, Terranova EFNMR lab data, and grouped-text-field studies"
 requires-python = ">=3.10"
 readme = "README.md"
 license = "MIT"
@@ -13,7 +13,7 @@ license-files = ["LICENSE"]
 authors = [
     { name = "zakgm2" },
 ]
-keywords = ["physics", "neuroscience", "fibre-photometry", "NIRS", "TDT", "EFNMR", "signal-processing"]
+keywords = ["physics", "neuroscience", "fibre-photometry", "NIRS", "TDT", "EFNMR", "signal-processing", "text-analysis", "nlp"]
 classifiers = [
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3.10",
@@ -27,6 +27,9 @@ dependencies = [
     "numpy",
     "scipy",
     "tdt",
+    "pandas",
+    "sentence-transformers",
+    "statsmodels",
 ]
 
 [project.urls]

zaksphysicslibrary-1.3.0/ZaksPhysicsLibrary.egg-info/requires.txt

@@ -1,3 +0,0 @@
-numpy
-scipy
-tdt