tpattern 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tpattern/__init__.py +36 -0
- tpattern/advisor.py +195 -0
- tpattern/ci.py +300 -0
- tpattern/contrast.py +90 -0
- tpattern/detect.py +334 -0
- tpattern/diagnostics.py +120 -0
- tpattern/guided.py +372 -0
- tpattern/io.py +214 -0
- tpattern/methods.py +132 -0
- tpattern/pattern.py +129 -0
- tpattern/randomise.py +206 -0
- tpattern/report.py +201 -0
- tpattern/significance.py +218 -0
- tpattern/synthetic.py +95 -0
- tpattern/viz.py +107 -0
- tpattern-0.1.0.dist-info/METADATA +211 -0
- tpattern-0.1.0.dist-info/RECORD +20 -0
- tpattern-0.1.0.dist-info/WHEEL +5 -0
- tpattern-0.1.0.dist-info/licenses/LICENSE +21 -0
- tpattern-0.1.0.dist-info/top_level.txt +1 -0
tpattern/significance.py
ADDED
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Surrogate-calibrated significance — the mathematically defensible keep/reject.
|
|
3
|
+
=============================================================================
|
|
4
|
+
|
|
5
|
+
The analytic critical-interval p-value is not trustworthy here (see
|
|
6
|
+
diagnostics.py: the data are strongly non-Poisson, and the detector selects the
|
|
7
|
+
best of many windows and pairs). So we do not threshold it directly. Instead we
|
|
8
|
+
calibrate every detected pattern against surrogate data run through the
|
|
9
|
+
*identical* detection pipeline. Because the surrogate detection performs the same
|
|
10
|
+
window/pair selection, the comparison is automatically corrected for selection
|
|
11
|
+
bias; because the surrogate preserves each event type's real marginal structure
|
|
12
|
+
(profile-preserving null), it is also corrected for the wrong independence
|
|
13
|
+
assumption.
|
|
14
|
+
|
|
15
|
+
Per Andy's decision we compute TWO statistics side by side:
|
|
16
|
+
|
|
17
|
+
1. Signature-matched empirical p (primary, interpretable). For each real pattern,
|
|
18
|
+
how often does the *same* pattern (identical signature) arise in the
|
|
19
|
+
surrogates with an occurrence count at least as high?
|
|
20
|
+
p_emp = (1 + #surrogates with that signature at N >= observed) / (1 + B)
|
|
21
|
+
This asks, directly, "could chance have produced THIS pattern this strongly?"
|
|
22
|
+
The +1/+1 is the standard Monte-Carlo correction (never reports p = 0).
|
|
23
|
+
|
|
24
|
+
2. Analytic strength (secondary). s = -log10(best-window binomial p), calibrated
|
|
25
|
+
the same way against the pooled surrogate strength distribution. Reported for
|
|
26
|
+
comparison / robustness, not as the primary decision.
|
|
27
|
+
|
|
28
|
+
Two error-rate controls, both stratified by pattern level (the search space, and
|
|
29
|
+
hence the null, differs by level):
|
|
30
|
+
|
|
31
|
+
* FWER (family-wise) — a real pattern is kept only if its empirical p survives a
|
|
32
|
+
Holm–Bonferroni step-down at level alpha across the patterns of its level.
|
|
33
|
+
Controls P(any false pattern) <= alpha. Conservative; confirmatory.
|
|
34
|
+
* FDR (false discovery rate) — Benjamini–Hochberg on the empirical p-values at
|
|
35
|
+
q_target. Controls the expected proportion of false patterns; more powerful,
|
|
36
|
+
for screening.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
from __future__ import annotations
|
|
40
|
+
|
|
41
|
+
import math
|
|
42
|
+
import random
|
|
43
|
+
from collections import defaultdict
|
|
44
|
+
from dataclasses import dataclass
|
|
45
|
+
|
|
46
|
+
import numpy as np
|
|
47
|
+
|
|
48
|
+
from .detect import Config, Engine
|
|
49
|
+
from .randomise import build_profiles, profile_surrogate, randomise_sample
|
|
50
|
+
from .pattern import Pattern
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _strength(p: Pattern) -> float:
|
|
54
|
+
pv = p.p_value if p.p_value and p.p_value > 0 else 1e-300
|
|
55
|
+
return -math.log10(pv)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@dataclass
|
|
59
|
+
class Calibrated:
|
|
60
|
+
pattern: Pattern
|
|
61
|
+
level: int
|
|
62
|
+
N: int
|
|
63
|
+
ci: tuple
|
|
64
|
+
p_emp: float = 1.0 # signature-matched empirical p (primary)
|
|
65
|
+
strength: float = 0.0 # analytic -log10(p) (secondary)
|
|
66
|
+
p_emp_strength: float = 1.0 # empirical p of the analytic strength
|
|
67
|
+
fwer_keep: bool = False # Holm on p_emp
|
|
68
|
+
fdr_q: float = 1.0 # Benjamini-Hochberg q on p_emp
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass
|
|
72
|
+
class CalibrationResult:
|
|
73
|
+
real: list[Calibrated]
|
|
74
|
+
surrogate_counts: dict # level -> list of per-surrogate pattern counts
|
|
75
|
+
B: int
|
|
76
|
+
alpha: float
|
|
77
|
+
q_target: float
|
|
78
|
+
null: str
|
|
79
|
+
|
|
80
|
+
def kept(self, method: str = "fdr") -> list[Calibrated]:
|
|
81
|
+
if method == "fwer":
|
|
82
|
+
return [c for c in self.real if c.fwer_keep]
|
|
83
|
+
return [c for c in self.real if c.fdr_q <= self.q_target]
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _benjamini_hochberg(items, get_p):
|
|
87
|
+
"""Assign BH q-values (in place ordering) to `items` by their p = get_p(x)."""
|
|
88
|
+
m = len(items)
|
|
89
|
+
if m == 0:
|
|
90
|
+
return {}
|
|
91
|
+
order = sorted(range(m), key=lambda i: get_p(items[i]))
|
|
92
|
+
q = [0.0] * m
|
|
93
|
+
running = 1.0
|
|
94
|
+
for rank in range(m - 1, -1, -1):
|
|
95
|
+
i = order[rank]
|
|
96
|
+
val = get_p(items[i]) * m / (rank + 1)
|
|
97
|
+
running = min(running, val)
|
|
98
|
+
q[i] = min(1.0, running)
|
|
99
|
+
return {id(items[i]): q[i] for i in range(m)}
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _holm_keep(items, get_p, alpha):
|
|
103
|
+
"""Holm–Bonferroni step-down; return set of ids kept at family alpha."""
|
|
104
|
+
m = len(items)
|
|
105
|
+
order = sorted(range(m), key=lambda i: get_p(items[i]))
|
|
106
|
+
kept = set()
|
|
107
|
+
for rank, i in enumerate(order):
|
|
108
|
+
if get_p(items[i]) <= alpha / (m - rank):
|
|
109
|
+
kept.add(id(items[i]))
|
|
110
|
+
else:
|
|
111
|
+
break
|
|
112
|
+
return kept
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def calibrate(observations, config: Config | None = None, *,
|
|
116
|
+
null: str = "profile", B: int = 200, alpha: float = 0.005,
|
|
117
|
+
q_target: float = 0.05, seed: int = 20260714) -> CalibrationResult:
|
|
118
|
+
"""Detect T-patterns, then test each one against a surrogate null.
|
|
119
|
+
|
|
120
|
+
This is the main analysis call: it runs the detection for you and calibrates
|
|
121
|
+
every detected pattern, so you do not need to call Engine.detect() first.
|
|
122
|
+
|
|
123
|
+
Parameters
|
|
124
|
+
----------
|
|
125
|
+
observations : list[Observation]
|
|
126
|
+
The sample, e.g. from read_table().
|
|
127
|
+
config : Config, optional
|
|
128
|
+
Detection settings (defaults match THEME). Use Config(min_lag=1) to require
|
|
129
|
+
a genuine temporal lag on frame-coded data. See advisor.recommend() for a
|
|
130
|
+
data-driven suggestion.
|
|
131
|
+
null : {"profile", "rotation", "shuffle"}
|
|
132
|
+
The surrogate null. "profile" preserves each event type's marginal timing and
|
|
133
|
+
destroys only cross-type coupling (recommended where timing is structured);
|
|
134
|
+
"rotation"/"shuffle" are THEME-style nulls.
|
|
135
|
+
B : int
|
|
136
|
+
Number of surrogate datasets. The empirical p-value cannot fall below
|
|
137
|
+
1/(B+1), so family-wise (FWER) claims need B in the thousands; FDR is fine
|
|
138
|
+
at B=200.
|
|
139
|
+
alpha, q_target : float
|
|
140
|
+
Family-wise level (Holm) and false-discovery target (Benjamini-Hochberg).
|
|
141
|
+
seed : int
|
|
142
|
+
Fixes the surrogate draw, so results are reproducible.
|
|
143
|
+
|
|
144
|
+
Returns
|
|
145
|
+
-------
|
|
146
|
+
CalibrationResult
|
|
147
|
+
Call .kept("fdr") or .kept("fwer") for the surviving patterns; each carries
|
|
148
|
+
an empirical p (.p_emp) and q-value (.fdr_q).
|
|
149
|
+
"""
|
|
150
|
+
config = config or Config()
|
|
151
|
+
|
|
152
|
+
real_pats = [p for p in Engine(observations, config).detect() if p.level >= 1]
|
|
153
|
+
real = [Calibrated(pattern=p, level=p.level, N=p.N, ci=p.ci,
|
|
154
|
+
strength=_strength(p)) for p in real_pats]
|
|
155
|
+
|
|
156
|
+
profiles = build_profiles(observations) if null == "profile" else None
|
|
157
|
+
np_rng = np.random.default_rng(seed)
|
|
158
|
+
|
|
159
|
+
# signature-matched: per real signature, how many surrogates reach its N
|
|
160
|
+
sig_N = {p.signature(): p.N for p in real_pats}
|
|
161
|
+
beat_count: dict[str, int] = defaultdict(int) # count-based
|
|
162
|
+
# analytic-strength: pooled surrogate strengths per level for empirical p
|
|
163
|
+
surr_strengths: dict[int, list[float]] = defaultdict(list)
|
|
164
|
+
surrogate_counts: dict[int, list[int]] = defaultdict(list)
|
|
165
|
+
track_levels = {c.level for c in real}
|
|
166
|
+
|
|
167
|
+
# Monte-Carlo core: generate B surrogate datasets and run the *identical*
|
|
168
|
+
# detector on each. Because detection performs the same window/pair selection
|
|
169
|
+
# on the surrogates as on the real data, comparing counts is automatically
|
|
170
|
+
# corrected for the detector's selection bias.
|
|
171
|
+
for b in range(B):
|
|
172
|
+
if null == "profile":
|
|
173
|
+
surr = profile_surrogate(observations, profiles, np_rng)
|
|
174
|
+
else:
|
|
175
|
+
surr = randomise_sample(observations, null, random.Random(seed + b))
|
|
176
|
+
pats = [p for p in Engine(surr, config).detect() if p.level >= 1]
|
|
177
|
+
|
|
178
|
+
# For this one surrogate, record the strongest occurrence count reached by
|
|
179
|
+
# each pattern signature, and collect the analytic strengths for each level.
|
|
180
|
+
best_by_sig: dict[str, int] = {}
|
|
181
|
+
per_level_strength: dict[int, list[float]] = defaultdict(list)
|
|
182
|
+
for p in pats:
|
|
183
|
+
sig = p.signature()
|
|
184
|
+
if p.N > best_by_sig.get(sig, 0):
|
|
185
|
+
best_by_sig[sig] = p.N
|
|
186
|
+
per_level_strength[p.level].append(_strength(p))
|
|
187
|
+
# A surrogate "beats" a real pattern when the *same* signature occurs in it
|
|
188
|
+
# at least as often as in the real data — i.e. chance reproduced it this
|
|
189
|
+
# strongly. Tally those hits per signature across all B surrogates.
|
|
190
|
+
for sig, n_obs in sig_N.items():
|
|
191
|
+
if best_by_sig.get(sig, 0) >= n_obs:
|
|
192
|
+
beat_count[sig] += 1
|
|
193
|
+
# Also pool the surrogate strengths per level, and record how many patterns
|
|
194
|
+
# this surrogate produced at each level (used for the analytic-strength p
|
|
195
|
+
# and for the reported surrogate-count distribution).
|
|
196
|
+
for lv in track_levels:
|
|
197
|
+
surr_strengths[lv].extend(per_level_strength.get(lv, []))
|
|
198
|
+
surrogate_counts[lv].append(len(per_level_strength.get(lv, [])))
|
|
199
|
+
|
|
200
|
+
# empirical p-values (Monte-Carlo, +1/+1 correction)
|
|
201
|
+
for c in real:
|
|
202
|
+
sig = c.pattern.signature()
|
|
203
|
+
c.p_emp = (1 + beat_count.get(sig, 0)) / (1 + B)
|
|
204
|
+
pool = surr_strengths.get(c.level, [])
|
|
205
|
+
ge = sum(1 for s in pool if s >= c.strength)
|
|
206
|
+
c.p_emp_strength = (1 + ge) / (1 + len(pool)) if pool else 1.0 / (1 + B)
|
|
207
|
+
|
|
208
|
+
# per-level FWER (Holm) and FDR (BH) on the primary empirical p
|
|
209
|
+
for lv in track_levels:
|
|
210
|
+
grp = [c for c in real if c.level == lv]
|
|
211
|
+
keep = _holm_keep(grp, lambda c: c.p_emp, alpha)
|
|
212
|
+
qmap = _benjamini_hochberg(grp, lambda c: c.p_emp)
|
|
213
|
+
for c in grp:
|
|
214
|
+
c.fwer_keep = id(c) in keep
|
|
215
|
+
c.fdr_q = qmap[id(c)]
|
|
216
|
+
|
|
217
|
+
return CalibrationResult(real=real, surrogate_counts=surrogate_counts,
|
|
218
|
+
B=B, alpha=alpha, q_target=q_target, null=null)
|
tpattern/synthetic.py
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Synthetic ground-truth validation.
|
|
3
|
+
==================================
|
|
4
|
+
|
|
5
|
+
A "nothing survives" result is only meaningful if the method is *able* to find
|
|
6
|
+
real patterns and *refrains* from inventing them. This module builds data where
|
|
7
|
+
the truth is known, so we can measure both:
|
|
8
|
+
|
|
9
|
+
* Power / sensitivity — when we plant a genuine A->B(->C) coupling in a fraction
|
|
10
|
+
of sequences, does the calibrated detector recover it, and at what strength?
|
|
11
|
+
* Calibration / specificity — on pure-null data (no planted coupling), does the
|
|
12
|
+
family-wise error rate stay at alpha (FWER) and the false-discovery proportion
|
|
13
|
+
at q (FDR)? If a method flags patterns in pure noise, it is broken.
|
|
14
|
+
|
|
15
|
+
Construction. Each sequence is a window [0, T]. Every event type has a marginal
|
|
16
|
+
temporal profile (where in the window it tends to fall) and a per-sequence count,
|
|
17
|
+
mirroring the real data's structure. Background events are placed independently
|
|
18
|
+
from their profiles — so there is NO cross-type coupling by construction. A
|
|
19
|
+
planted pattern adds, in a chosen fraction of sequences, events A, B (, C) at a
|
|
20
|
+
fixed inter-event lag plus jitter — a genuine, known temporal coupling. Anything
|
|
21
|
+
the detector reports other than the planted pattern is, by construction, a false
|
|
22
|
+
positive.
|
|
23
|
+
|
|
24
|
+
This is deliberately generated independently of the tpattern engine so it is a
|
|
25
|
+
true external test, not a tautology.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
from dataclasses import dataclass, field
|
|
31
|
+
|
|
32
|
+
import numpy as np
|
|
33
|
+
|
|
34
|
+
from .io import Observation
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass
|
|
38
|
+
class SyntheticSpec:
|
|
39
|
+
n_seq: int = 300
|
|
40
|
+
T: int = 6000 # window length (ms)
|
|
41
|
+
bg_types: int = 8 # number of background event types
|
|
42
|
+
bg_rate: float = 2.0 # mean background events per type per seq
|
|
43
|
+
# planted pattern: sequence of (label, lag_ms_from_previous)
|
|
44
|
+
planted: list = field(default_factory=lambda: [("A", 0), ("B", 1500)])
|
|
45
|
+
plant_fraction: float = 0.15 # fraction of sequences carrying it
|
|
46
|
+
jitter: int = 200 # +/- ms jitter on planted lags
|
|
47
|
+
seed: int = 7
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _profile_time(rng, T, kind):
|
|
51
|
+
"""Draw a within-window time for a background event given a profile kind."""
|
|
52
|
+
if kind == "early":
|
|
53
|
+
u = rng.beta(2, 5)
|
|
54
|
+
elif kind == "late":
|
|
55
|
+
u = rng.beta(5, 2)
|
|
56
|
+
else:
|
|
57
|
+
u = rng.random()
|
|
58
|
+
return int(u * T)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def make_dataset(spec: SyntheticSpec):
|
|
62
|
+
"""Return (observations, truth_signature) for one synthetic spec."""
|
|
63
|
+
rng = np.random.default_rng(spec.seed)
|
|
64
|
+
kinds = ["early", "mid", "late"]
|
|
65
|
+
type_kind = {f"bg{i}": kinds[i % 3] for i in range(spec.bg_types)}
|
|
66
|
+
|
|
67
|
+
obs = []
|
|
68
|
+
n_plant = int(round(spec.n_seq * spec.plant_fraction))
|
|
69
|
+
plant_flags = np.array([i < n_plant for i in range(spec.n_seq)])
|
|
70
|
+
rng.shuffle(plant_flags)
|
|
71
|
+
|
|
72
|
+
for s in range(spec.n_seq):
|
|
73
|
+
events = []
|
|
74
|
+
# background: each type gets Poisson(bg_rate) events from its profile
|
|
75
|
+
for ev, kind in type_kind.items():
|
|
76
|
+
c = rng.poisson(spec.bg_rate)
|
|
77
|
+
for _ in range(c):
|
|
78
|
+
events.append((_profile_time(rng, spec.T, kind), ev))
|
|
79
|
+
# planted coupling in a fraction of sequences
|
|
80
|
+
if plant_flags[s]:
|
|
81
|
+
anchor = int(rng.uniform(0.1, 0.6) * spec.T)
|
|
82
|
+
t = anchor
|
|
83
|
+
for lbl, lag in spec.planted:
|
|
84
|
+
t = t + lag + int(rng.integers(-spec.jitter, spec.jitter + 1))
|
|
85
|
+
events.append((max(0, min(spec.T, t)), lbl))
|
|
86
|
+
events.sort()
|
|
87
|
+
obs.append(Observation(name=f"syn{s}", start=0, end=spec.T, events=events))
|
|
88
|
+
|
|
89
|
+
truth = "(" * (len(spec.planted) - 1)
|
|
90
|
+
# build the expected signature the same way Pattern.signature would (left-assoc)
|
|
91
|
+
labels = [lbl for lbl, _ in spec.planted]
|
|
92
|
+
sig = labels[0]
|
|
93
|
+
for lbl in labels[1:]:
|
|
94
|
+
sig = f"({sig} {lbl})"
|
|
95
|
+
return obs, sig
|
tpattern/viz.py
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Visual output for detected T-patterns — clean, publication-ready figures.
|
|
3
|
+
|
|
4
|
+
The centrepiece is `pattern_dendrogram`: a T-pattern is a binary tree of
|
|
5
|
+
critical-interval links, and this draws it as a proper dendrogram — event types
|
|
6
|
+
as leaves, each internal join annotated with its critical interval [d1, d2] (the
|
|
7
|
+
time window within which the right side follows the left). THEME draws this as a
|
|
8
|
+
cramped detection tree; here it is a clean, labelled figure.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import matplotlib
|
|
14
|
+
matplotlib.use("Agg")
|
|
15
|
+
import matplotlib.pyplot as plt
|
|
16
|
+
|
|
17
|
+
from .pattern import Pattern
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _layout(p: Pattern, leaves: list, pos: dict):
|
|
21
|
+
"""Assign (x, depth) to every node. Leaves get sequential x; internal nodes
|
|
22
|
+
sit at the mid-x of their children, depth = 1 + max child depth."""
|
|
23
|
+
if p.is_terminal:
|
|
24
|
+
x = len(leaves)
|
|
25
|
+
leaves.append(p.event)
|
|
26
|
+
pos[id(p)] = (x, 0)
|
|
27
|
+
return x, 0
|
|
28
|
+
lx, ld = _layout(p.left, leaves, pos)
|
|
29
|
+
rx, rd = _layout(p.right, leaves, pos)
|
|
30
|
+
x = (lx + rx) / 2.0
|
|
31
|
+
d = 1 + max(ld, rd)
|
|
32
|
+
pos[id(p)] = (x, d)
|
|
33
|
+
return x, d
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _draw(p: Pattern, pos: dict, ax, ci_unit: str):
|
|
37
|
+
"""Recursively draw dendrogram elbows from each internal node to its children."""
|
|
38
|
+
if p.is_terminal:
|
|
39
|
+
return
|
|
40
|
+
x, d = pos[id(p)]
|
|
41
|
+
for child in (p.left, p.right):
|
|
42
|
+
cx, cd = pos[id(child)]
|
|
43
|
+
# elbow: up from child to this node's depth, then across
|
|
44
|
+
ax.plot([cx, cx], [cd, d], color="#34495e", lw=1.4)
|
|
45
|
+
ax.plot([cx, x], [d, d], color="#34495e", lw=1.4)
|
|
46
|
+
_draw(child, pos, ax, ci_unit)
|
|
47
|
+
# annotate the critical interval on the join
|
|
48
|
+
if p.ci is not None:
|
|
49
|
+
d1, d2 = p.ci
|
|
50
|
+
ax.annotate(f"[{d1},{d2}]{ci_unit}", (x, d), textcoords="offset points",
|
|
51
|
+
xytext=(0, 4), ha="center", fontsize=7, color="#c0392b")
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def pattern_dendrogram(pattern: Pattern, title: str | None = None,
|
|
55
|
+
ci_unit: str = "", outfile: str | None = None, ax=None):
|
|
56
|
+
"""Draw one T-pattern as a dendrogram. Returns the Axes.
|
|
57
|
+
|
|
58
|
+
Leaves (event types) are labelled along the bottom in temporal order; each
|
|
59
|
+
internal join shows its critical interval. `pattern.N` and significance can
|
|
60
|
+
be put in the title.
|
|
61
|
+
"""
|
|
62
|
+
leaves: list = []
|
|
63
|
+
pos: dict = {}
|
|
64
|
+
_layout(pattern, leaves, pos)
|
|
65
|
+
|
|
66
|
+
own = ax is None
|
|
67
|
+
if own:
|
|
68
|
+
fig, ax = plt.subplots(figsize=(max(4, 1.1 * len(leaves)), 3.2))
|
|
69
|
+
_draw(pattern, pos, ax, ci_unit)
|
|
70
|
+
|
|
71
|
+
# leaf labels
|
|
72
|
+
for i, name in enumerate(leaves):
|
|
73
|
+
ax.plot([i], [0], "o", color="#2980b9", ms=6)
|
|
74
|
+
ax.annotate(name, (i, 0), textcoords="offset points", xytext=(0, -8),
|
|
75
|
+
ha="right", va="top", rotation=35, fontsize=8)
|
|
76
|
+
|
|
77
|
+
ax.set_xlim(-0.7, len(leaves) - 0.3)
|
|
78
|
+
ax.set_ylim(-1.2, pattern.level + 0.6)
|
|
79
|
+
ax.set_yticks(range(pattern.level + 1))
|
|
80
|
+
ax.set_ylabel("level")
|
|
81
|
+
ax.set_xticks([])
|
|
82
|
+
for s in ("top", "right", "bottom"):
|
|
83
|
+
ax.spines[s].set_visible(False)
|
|
84
|
+
if title:
|
|
85
|
+
ax.set_title(title, fontsize=9, wrap=True)
|
|
86
|
+
|
|
87
|
+
if own and outfile:
|
|
88
|
+
plt.tight_layout(); plt.savefig(outfile, dpi=150); plt.close()
|
|
89
|
+
return ax
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def patterns_overview(patterns: list, outfile: str, max_rows: int = 8,
|
|
93
|
+
ci_unit: str = "", by=lambda p: -p.N):
|
|
94
|
+
"""A grid of the top pattern dendrograms (by `by`, default most frequent)."""
|
|
95
|
+
comps = sorted([p for p in patterns if p.level >= 1], key=by)[:max_rows]
|
|
96
|
+
n = len(comps)
|
|
97
|
+
if not n:
|
|
98
|
+
return
|
|
99
|
+
cols = 2
|
|
100
|
+
rows = (n + cols - 1) // cols
|
|
101
|
+
fig, axes = plt.subplots(rows, cols, figsize=(12, 3.0 * rows))
|
|
102
|
+
axes = axes.flatten() if n > 1 else [axes]
|
|
103
|
+
for ax, p in zip(axes, comps):
|
|
104
|
+
pattern_dendrogram(p, title=f"N={p.N}, level {p.level}", ci_unit=ci_unit, ax=ax)
|
|
105
|
+
for ax in axes[n:]:
|
|
106
|
+
ax.axis("off")
|
|
107
|
+
plt.tight_layout(); plt.savefig(outfile, dpi=150); plt.close()
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: tpattern
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Open, auditable temporal & sequential pattern analysis (T-patterns) for sport and behaviour
|
|
5
|
+
Author-email: Andrew Callaway <acallaway@bournemouth.ac.uk>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/ajcallaway/TPattern
|
|
8
|
+
Project-URL: Repository, https://github.com/ajcallaway/TPattern
|
|
9
|
+
Project-URL: Archive (DOI), https://doi.org/10.5281/zenodo.21397543
|
|
10
|
+
Project-URL: Tagging tool (OpenTag), https://opentag.studio
|
|
11
|
+
Keywords: t-pattern,THEME,temporal,sequence,sport,behaviour,notation analysis
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
19
|
+
Classifier: Operating System :: OS Independent
|
|
20
|
+
Classifier: Intended Audience :: Science/Research
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering
|
|
22
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
23
|
+
Requires-Python: >=3.9
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
License-File: LICENSE
|
|
26
|
+
Requires-Dist: numpy>=1.21
|
|
27
|
+
Requires-Dist: scipy>=1.7
|
|
28
|
+
Requires-Dist: matplotlib>=3.4
|
|
29
|
+
Provides-Extra: test
|
|
30
|
+
Requires-Dist: pytest>=7; extra == "test"
|
|
31
|
+
Provides-Extra: gui
|
|
32
|
+
Requires-Dist: ipywidgets>=8; extra == "gui"
|
|
33
|
+
Requires-Dist: pandas>=1.3; extra == "gui"
|
|
34
|
+
Requires-Dist: openpyxl>=3; extra == "gui"
|
|
35
|
+
Dynamic: license-file
|
|
36
|
+
|
|
37
|
+
# tpattern — open, auditable analysis of recurring patterns in event sequences
|
|
38
|
+
|
|
39
|
+
[](https://colab.research.google.com/github/ajcallaway/TPattern/blob/main/colab/tpattern_guided_analysis.ipynb)
|
|
40
|
+
[](https://doi.org/10.5281/zenodo.21397543)
|
|
41
|
+
[](LICENSE)
|
|
42
|
+
|
|
43
|
+
`tpattern` detects recurring temporal patterns (T-patterns; Magnusson, 2000) in
|
|
44
|
+
**any sequence of timed, labelled events** — behaviour, sport, clinical or
|
|
45
|
+
physiological logs, interaction data — validates them properly against chance, and
|
|
46
|
+
produces clean, publication-ready output (tables, figures, pattern dendrograms). It
|
|
47
|
+
is a free, transparent, reproducible alternative to the THEME software, and the
|
|
48
|
+
analysis companion to the [OpenTag.Studio](https://opentag.studio) tagging tool.
|
|
49
|
+
|
|
50
|
+
## Two ways to use it
|
|
51
|
+
|
|
52
|
+
- 🚀 **Google Colab — no coding.** [**Open the guided notebook in Colab**](https://colab.research.google.com/github/ajcallaway/TPattern/blob/main/colab/tpattern_guided_analysis.ipynb): upload your events, follow the guided steps, download a report with a plain-English verdict for every pattern and a paste-ready Methods paragraph. See [`colab/`](colab/).
|
|
53
|
+
- 🐍 **Python library.** `pip install` and script it — see [Quickstart](#quickstart) and [`examples/`](examples/).
|
|
54
|
+
|
|
55
|
+
Bring your own data (one row per event — see [`SCHEMA.md`](SCHEMA.md)) or try the
|
|
56
|
+
included [`data/`](data/) example.
|
|
57
|
+
|
|
58
|
+
## Why it exists
|
|
59
|
+
|
|
60
|
+
T-pattern analysis finds behaviourally meaningful structure that event counts and
|
|
61
|
+
simple transition tables miss. It is most established through Magnusson's THEME
|
|
62
|
+
software, the reference implementation. `tpattern` is an open, complementary
|
|
63
|
+
implementation that reproduces THEME's detection and adds a reproducible chance
|
|
64
|
+
model and clear reporting:
|
|
65
|
+
|
|
66
|
+
- **reproduces** THEME's core detection (critical intervals, Nx/T baseline,
|
|
67
|
+
completeness competition) — validated to the pattern for published datasets;
|
|
68
|
+
- **adds a chance model** — every pattern is tested against a surrogate null, with
|
|
69
|
+
p-values **corrected for repeated testing** (false-discovery rate and
|
|
70
|
+
family-wise);
|
|
71
|
+
- **handles concurrency explicitly** — events sharing a timestamp are treated as
|
|
72
|
+
co-occurrence rather than sequence;
|
|
73
|
+
- **reports clearly** — tidy tables, effect sizes with confidence intervals,
|
|
74
|
+
purposeful figures, and clean **dendrograms** of the detected patterns.
|
|
75
|
+
|
|
76
|
+
## Installation
|
|
77
|
+
|
|
78
|
+
From source (until published on PyPI):
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
git clone https://github.com/ajcallaway/TPattern.git
|
|
82
|
+
cd tpattern
|
|
83
|
+
pip install -e .
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Requires Python ≥ 3.9 (depends on numpy, scipy, matplotlib).
|
|
87
|
+
|
|
88
|
+
## Quickstart
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
from tpattern import read_table, Config, recommend, calibrate, report, methods_text
|
|
92
|
+
|
|
93
|
+
# 1. Load your events — one row per event (observation, event, start).
|
|
94
|
+
# Pass time_unit="ms" if your times are in milliseconds (default is seconds).
|
|
95
|
+
# See SCHEMA.md for the (simple) required format.
|
|
96
|
+
observations = read_table("events.csv", time_unit="s")
|
|
97
|
+
|
|
98
|
+
# 2. Let the tool inspect the data and recommend the method — which surrogate
|
|
99
|
+
# null, whether to require a genuine lag, which error control — with reasons.
|
|
100
|
+
print(recommend(observations))
|
|
101
|
+
|
|
102
|
+
# 3. Detect and calibrate in one call: every pattern is tested against a
|
|
103
|
+
# surrogate null with correction for repeated testing.
|
|
104
|
+
# (calibrate() runs the detection for you — you don't call detect() first.)
|
|
105
|
+
result = calibrate(observations, Config(), null="profile", B=200, q_target=0.05)
|
|
106
|
+
survivors = result.kept("fdr") # patterns that survive false-discovery control
|
|
107
|
+
|
|
108
|
+
# 4. Report: a tidy table (with q-values), pattern dendrograms and a summary —
|
|
109
|
+
# plus a paste-ready Methods paragraph stating every setting and what it does.
|
|
110
|
+
report(result, "report_out", title="My analysis")
|
|
111
|
+
print(methods_text(Config(), observations=observations, calibration=result))
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
Two runnable examples:
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
pip install -e .
|
|
118
|
+
python examples/quickstart.py # no data needed (synthetic ground-truth demo)
|
|
119
|
+
python examples/reproduce_worldcup.py # a real read_table + full analysis on shipped data
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
Just want the detected patterns without calibration? `Engine(observations,
|
|
123
|
+
Config()).detect()` returns them directly.
|
|
124
|
+
|
|
125
|
+
**Comparing two groups** (winner vs loser, goal vs non-goal, condition A vs B)?
|
|
126
|
+
`group_contrast(observations, group_of)` tests each pattern's occurrence between
|
|
127
|
+
the groups (Fisher's exact test, odds ratio + 95% CI) and feeds `forest_plot`.
|
|
128
|
+
|
|
129
|
+
## The workflow
|
|
130
|
+
|
|
131
|
+
```
|
|
132
|
+
tag events export analyse report
|
|
133
|
+
(OpenTag / Sportscode) ──────▶ standard ──▶ tpattern ──▶ tables · figures · dendrograms
|
|
134
|
+
table (detect · validate ·
|
|
135
|
+
calibrate)
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
1. **Tag** your video/events in any coding tool.
|
|
139
|
+
2. **Export** to the standard input structure — one row per event
|
|
140
|
+
(`observation, event, start`, plus optional `end`, descriptor and group
|
|
141
|
+
columns). See [`SCHEMA.md`](SCHEMA.md).
|
|
142
|
+
3. **Set up your coding** so the export is analysis-ready — one event per action,
|
|
143
|
+
explicit outcomes, unit boundaries. See [`CODING_GUIDE.md`](CODING_GUIDE.md).
|
|
144
|
+
4. **Detect** T-patterns, **validate** them against a surrogate null with corrected
|
|
145
|
+
significance (`recommend` advises the settings; `calibrate` runs it), and
|
|
146
|
+
**report** (tables, dendrograms, and a `methods_text` Methods paragraph).
|
|
147
|
+
|
|
148
|
+
## Relationship to OpenTag.Studio
|
|
149
|
+
|
|
150
|
+
[OpenTag.Studio](https://opentag.studio) is a free, browser-based sports video
|
|
151
|
+
tagging tool. It is the natural front end for `tpattern`: tag your footage, export
|
|
152
|
+
a session, and analyse it here — no proprietary software anywhere in the pipeline.
|
|
153
|
+
OpenTag's **THEME & GSEQ setup guide** covers how to design a coding window so the
|
|
154
|
+
export is analysis-ready, and that guidance applies directly to `tpattern`
|
|
155
|
+
(`CODING_GUIDE.md` is the short bridge). Together they form a complete, open
|
|
156
|
+
tag → analysis pipeline.
|
|
157
|
+
|
|
158
|
+
## Reproducing the published results
|
|
159
|
+
|
|
160
|
+
The derived 2022 World Cup event sequences ship in [`data/`](data/), so the
|
|
161
|
+
published analyses reproduce from this repository alone:
|
|
162
|
+
|
|
163
|
+
```bash
|
|
164
|
+
python examples/reproduce_worldcup.py # concordance, exclusion, concurrency (~5 s)
|
|
165
|
+
python examples/reproduce_worldcup.py --calibrate # also the surrogate null (~10 min)
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
This regenerates the tpattern-vs-THEME pattern counts by level, the interception
|
|
169
|
+
exclusion divergence, the co-occurrence-vs-sequence analysis, and how many patterns
|
|
170
|
+
survive the surrogate null. All seeded and deterministic. See
|
|
171
|
+
[`data/README.md`](data/README.md) for provenance and how the data was derived.
|
|
172
|
+
|
|
173
|
+
## Documentation
|
|
174
|
+
|
|
175
|
+
- [`SCHEMA.md`](SCHEMA.md) — the input data structure (what your export must contain).
|
|
176
|
+
- [`CODING_GUIDE.md`](CODING_GUIDE.md) — how to set up a coding window to produce it.
|
|
177
|
+
- [`data/`](data/) — the derived World Cup sequences, in the canonical format.
|
|
178
|
+
- [`examples/`](examples/) — a runnable quickstart and the reproduction script.
|
|
179
|
+
|
|
180
|
+
## Tests
|
|
181
|
+
|
|
182
|
+
```bash
|
|
183
|
+
pip install -e .[test]
|
|
184
|
+
pytest -q
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
Both the synthetic ground-truth tests and the THEME reproduction tests run
|
|
188
|
+
everywhere — the latter check the detector still reproduces THEME's published
|
|
189
|
+
occurrence counts to the exact N.
|
|
190
|
+
|
|
191
|
+
## Citation
|
|
192
|
+
|
|
193
|
+
If you use `tpattern` in research, please cite the software (a machine-readable
|
|
194
|
+
version is in [`CITATION.cff`](CITATION.cff)):
|
|
195
|
+
|
|
196
|
+
Callaway, A. (2026). *tpattern: open, auditable temporal & sequential analysis for
|
|
197
|
+
sport and behaviour* [Computer software]. Zenodo.
|
|
198
|
+
https://doi.org/10.5281/zenodo.21397543
|
|
199
|
+
|
|
200
|
+
This DOI is permanent and always resolves to the current release. To pin a specific
|
|
201
|
+
version instead, use that release's own DOI, listed on the Zenodo record.
|
|
202
|
+
|
|
203
|
+
And the method:
|
|
204
|
+
|
|
205
|
+
Magnusson, M. S. (2000). Discovering hidden time patterns in behavior: T-patterns
|
|
206
|
+
and their detection. *Behavior Research Methods, Instruments, & Computers*, 32(1),
|
|
207
|
+
93–110. https://doi.org/10.3758/BF03200792
|
|
208
|
+
|
|
209
|
+
## License
|
|
210
|
+
|
|
211
|
+
MIT — see [`LICENSE`](LICENSE).
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
tpattern/__init__.py,sha256=8S-AKvLjy8XRYkhyTAE5uuNppoR8ETFN42yHy2ayKqI,1408
|
|
2
|
+
tpattern/advisor.py,sha256=72g-0Q1ANwLmV-lN3dsmCHGKVyvs2YXZCDBSsm4ATLw,10138
|
|
3
|
+
tpattern/ci.py,sha256=SZQ-ljLdg5LTTxsuuHbccOPEqI7cze0Ldv9K3IiDvcU,12896
|
|
4
|
+
tpattern/contrast.py,sha256=VHxKoFNJtauouyAsn0awtvtzukrfueFeu3nVbT120dM,3808
|
|
5
|
+
tpattern/detect.py,sha256=0iRyzOOJARXIId23K77R23kJhKxORGG9sRnuXLT4qdU,16449
|
|
6
|
+
tpattern/diagnostics.py,sha256=i2Os9NQIZFngWzZGaoZ972CWDIcyKZjvdUYw4vSW310,5207
|
|
7
|
+
tpattern/guided.py,sha256=pPhaHYwh2Yxf9Z2hl861HRcbboEE3SzEB7vOjMV3SQo,18676
|
|
8
|
+
tpattern/io.py,sha256=7TTxDcXz4Mx9Edp-tOBPyoUXZhF8R1i4z4dhXfFtMAM,8823
|
|
9
|
+
tpattern/methods.py,sha256=HotEOb6KuMaSpeOUkDXHD9p4x0DesA7uK41GUtxodQg,7095
|
|
10
|
+
tpattern/pattern.py,sha256=XM0NR_8rvQ61LqWceHDwlYLKi6h-TTm0oUF92SbNKrg,5122
|
|
11
|
+
tpattern/randomise.py,sha256=32wG2p5YKYc7UREq-NJsoYUqJ_iudmM7T92rhISmdeA,8922
|
|
12
|
+
tpattern/report.py,sha256=sHoYBifYz4Y3r9XE_CG9jorjadpJ1lhxHt9qSbunFxk,8298
|
|
13
|
+
tpattern/significance.py,sha256=kxFya9vLzlUzQ_4p5q-2fXGyQSdMXRX8pYwA6g_F6VM,9332
|
|
14
|
+
tpattern/synthetic.py,sha256=acSi1FKdF5TJPiRJdR4FIGD2H1CHhIL3C3dz537p5AA,3826
|
|
15
|
+
tpattern/viz.py,sha256=rmrzdWWcUhVtbC0kiOqadW1iwXiDPm5ZXLjhn4J6nuI,3899
|
|
16
|
+
tpattern-0.1.0.dist-info/licenses/LICENSE,sha256=mwn2NKwnIh3O8I2cT9Fd9HAw9sGgdMixJ3BSvE4sOlE,1072
|
|
17
|
+
tpattern-0.1.0.dist-info/METADATA,sha256=EqTmPsjaJvtBDTmlzzHcmTLFJnPIAmERB4Ropo3Pgno,9666
|
|
18
|
+
tpattern-0.1.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
|
|
19
|
+
tpattern-0.1.0.dist-info/top_level.txt,sha256=PTSDtDgp6IheACBj7DpFWRM6BYG2Ms0cxnuBy4yK4aA,9
|
|
20
|
+
tpattern-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Andrew Callaway
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|