python-hiddenmarkov 0.3.0__cp310-cp310-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hiddenmarkov/__init__.py +852 -0
- hiddenmarkov/_hmm.c +33227 -0
- hiddenmarkov/_hmm.cp310-win_amd64.pyd +0 -0
- hiddenmarkov/_hmm.pyi +10 -0
- hiddenmarkov/_hmm.pyx +90 -0
- hiddenmarkov/py.typed +0 -0
- hiddenmarkov/utils.py +55 -0
- python_hiddenmarkov-0.3.0.dist-info/METADATA +84 -0
- python_hiddenmarkov-0.3.0.dist-info/RECORD +12 -0
- python_hiddenmarkov-0.3.0.dist-info/WHEEL +5 -0
- python_hiddenmarkov-0.3.0.dist-info/licenses/LICENSE +21 -0
- python_hiddenmarkov-0.3.0.dist-info/top_level.txt +2 -0
hiddenmarkov/__init__.py
ADDED
|
@@ -0,0 +1,852 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
"""
|
|
4
|
+
Hidden Markov Models
|
|
5
|
+
|
|
6
|
+
This module contains case classes to define Hidden Markov Models.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
from importlib.metadata import version, PackageNotFoundError
|
|
12
|
+
|
|
13
|
+
from collections import defaultdict
|
|
14
|
+
from typing import Any, Callable, ClassVar, Iterable, List, Optional, Tuple
|
|
15
|
+
|
|
16
|
+
from scipy.special import logsumexp
|
|
17
|
+
|
|
18
|
+
from .utils import inverted_softmax, softmax
|
|
19
|
+
|
|
20
|
+
try:
|
|
21
|
+
from ._hmm import viterbi_loop as _viterbi_loop
|
|
22
|
+
_HAS_CYTHON = True
|
|
23
|
+
except ImportError: # pragma: no cover
|
|
24
|
+
_HAS_CYTHON = False
|
|
25
|
+
|
|
26
|
+
# Type alias for a single windowed probability model entry.
|
|
27
|
+
_ProbModelEntry = Tuple[Callable[..., Any], int]
|
|
28
|
+
|
|
29
|
+
# define a version variable
|
|
30
|
+
try:
|
|
31
|
+
__version__ = version("python-hiddenmarkov")
|
|
32
|
+
except PackageNotFoundError: # pragma: no cover
|
|
33
|
+
__version__ = "unknown"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class TransitionModel:
|
|
37
|
+
"""
|
|
38
|
+
Base class for implementing a Transition Model
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
init_probabilities: ClassVar[np.ndarray]
|
|
42
|
+
n_states: int
|
|
43
|
+
|
|
44
|
+
def __init__(self, use_log_probabilities: bool = True) -> None:
|
|
45
|
+
self.use_log_probabilities: bool = use_log_probabilities
|
|
46
|
+
|
|
47
|
+
def __call__(
|
|
48
|
+
self,
|
|
49
|
+
i: Optional[int] = None,
|
|
50
|
+
j: Optional[int] = None,
|
|
51
|
+
*args: Any,
|
|
52
|
+
**kwargs: Any,
|
|
53
|
+
) -> np.ndarray:
|
|
54
|
+
raise NotImplementedError
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class ObservationModel:
|
|
58
|
+
"""
|
|
59
|
+
Base class for implementing an Observation Model
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
def __init__(self, use_log_probabilities: bool = True) -> None:
|
|
63
|
+
self.use_log_probabilities: bool = use_log_probabilities
|
|
64
|
+
|
|
65
|
+
def __call__(self, observation: Any, *args: Any, **kwargs: Any) -> np.ndarray:
|
|
66
|
+
raise NotImplementedError()
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class HiddenMarkovModel:
|
|
70
|
+
"""
|
|
71
|
+
Hidden Markov Model
|
|
72
|
+
|
|
73
|
+
Parameters
|
|
74
|
+
----------
|
|
75
|
+
observation_model : ObservationModel
|
|
76
|
+
A model for computiong the observation (emission) probabilities.
|
|
77
|
+
transition_model: TransitionModel
|
|
78
|
+
A model for computing the transition probabilities.
|
|
79
|
+
state_space: iterable (optional)
|
|
80
|
+
Labels of the states (e.g., a list of strings containing
|
|
81
|
+
the names of each state).
|
|
82
|
+
forward: {"optimized", "standard"}
|
|
83
|
+
The implementation of the forward algorithm step used by
|
|
84
|
+
:meth:`forward_algorithm_step`. ``"optimized"`` (default)
|
|
85
|
+
restricts the vector-matrix product to the states that currently
|
|
86
|
+
have non-zero probability and to the columns reachable from them
|
|
87
|
+
(see :func:`forward_algorithm_step_optimized`); it requires a
|
|
88
|
+
:class:`ConstantTransitionModel`, and for any other transition
|
|
89
|
+
model the standard implementation is used instead. ``"standard"``
|
|
90
|
+
uses :func:`forward_algorithm_step`. Both give numerically
|
|
91
|
+
equivalent results.
|
|
92
|
+
|
|
93
|
+
Attributes
|
|
94
|
+
----------
|
|
95
|
+
observation_model: ObservationModel
|
|
96
|
+
transition_model: TransitionModel
|
|
97
|
+
n_states: int
|
|
98
|
+
Number of states
|
|
99
|
+
state_space: np.array
|
|
100
|
+
|
|
101
|
+
"""
|
|
102
|
+
|
|
103
|
+
def __init__(
|
|
104
|
+
self,
|
|
105
|
+
observation_model: ObservationModel,
|
|
106
|
+
transition_model: TransitionModel,
|
|
107
|
+
state_space: Optional[Iterable] = None,
|
|
108
|
+
forward: str = "optimized",
|
|
109
|
+
) -> None:
|
|
110
|
+
super().__init__()
|
|
111
|
+
if forward not in _FORWARD_ALGORITHMS:
|
|
112
|
+
raise ValueError(
|
|
113
|
+
"`forward` needs to be 'optimized' or 'standard' "
|
|
114
|
+
f"but is {forward!r}"
|
|
115
|
+
)
|
|
116
|
+
self.forward: str = forward
|
|
117
|
+
self._forward_step: Callable[..., np.ndarray] = _FORWARD_ALGORITHMS[forward]
|
|
118
|
+
if forward == "optimized" and not isinstance(
|
|
119
|
+
transition_model, ConstantTransitionModel
|
|
120
|
+
):
|
|
121
|
+
# The optimization relies on the structure of a constant
|
|
122
|
+
# transition matrix; other models use the standard algorithm.
|
|
123
|
+
self._forward_step = forward_algorithm_step
|
|
124
|
+
self.observation_model: ObservationModel = observation_model
|
|
125
|
+
self.transition_model: TransitionModel = transition_model
|
|
126
|
+
self.n_states: int = self.transition_model.n_states
|
|
127
|
+
if state_space is not None:
|
|
128
|
+
self.state_space: np.ndarray = np.asarray(state_space)
|
|
129
|
+
else:
|
|
130
|
+
self.state_space: np.ndarray = np.arange(self.n_states)
|
|
131
|
+
|
|
132
|
+
self.forward_variable: Optional[np.ndarray] = None
|
|
133
|
+
|
|
134
|
+
def find_best_sequence(
|
|
135
|
+
self,
|
|
136
|
+
observations: Iterable[Any],
|
|
137
|
+
log_probabilities: bool = True,
|
|
138
|
+
viterbi: str = "optimized",
|
|
139
|
+
) -> Tuple[np.ndarray, float]:
|
|
140
|
+
"""
|
|
141
|
+
Find the best sequence of hidden states given a sequence of
|
|
142
|
+
observations using the viterbi algorithm
|
|
143
|
+
|
|
144
|
+
Parameters
|
|
145
|
+
----------
|
|
146
|
+
observations : Iterable
|
|
147
|
+
The sequence of observations.
|
|
148
|
+
log_probabilities : bool
|
|
149
|
+
If True, log probabilities will be used.
|
|
150
|
+
viterbi : {"optimized", "windowed", "naive"}
|
|
151
|
+
The implementation of the viterbi algorithm.
|
|
152
|
+
|
|
153
|
+
Returns
|
|
154
|
+
-------
|
|
155
|
+
best_sequence : np.ndarray
|
|
156
|
+
The best sequence of hidden states
|
|
157
|
+
sequence_likelihood : float
|
|
158
|
+
The probability (or log probability if `log_probabilities=True`)
|
|
159
|
+
of the sequence of observations.
|
|
160
|
+
"""
|
|
161
|
+
if viterbi == "optimized":
|
|
162
|
+
viterbi_fun = viterbi_algorithm
|
|
163
|
+
elif viterbi == "windowed":
|
|
164
|
+
viterbi_fun = viterbi_algorithm_windowed
|
|
165
|
+
elif viterbi == "naive":
|
|
166
|
+
viterbi_fun = viterbi_algorithm_naive
|
|
167
|
+
else:
|
|
168
|
+
raise ValueError(
|
|
169
|
+
"`viterbi` needs to be 'optimized', 'windowed', or 'naive' "
|
|
170
|
+
f"but is {viterbi}"
|
|
171
|
+
)
|
|
172
|
+
# warnings.warn("viterbi needs to be 'optimized', 'windowed', or 'naive'")
|
|
173
|
+
# return
|
|
174
|
+
best_sequence, sequence_likelihood = viterbi_fun(
|
|
175
|
+
hmm=self,
|
|
176
|
+
observations=observations,
|
|
177
|
+
log_probabilities=log_probabilities,
|
|
178
|
+
)
|
|
179
|
+
return best_sequence, sequence_likelihood
|
|
180
|
+
|
|
181
|
+
def forward_algorithm_step(
|
|
182
|
+
self,
|
|
183
|
+
observation: Any,
|
|
184
|
+
log_probabilities: bool = False,
|
|
185
|
+
) -> int:
|
|
186
|
+
"""
|
|
187
|
+
Find the hidden state that has the maximal probability
|
|
188
|
+
given the current observation.
|
|
189
|
+
|
|
190
|
+
Parameters
|
|
191
|
+
----------
|
|
192
|
+
observation : Any
|
|
193
|
+
The current observation
|
|
194
|
+
log_probabilities: bool
|
|
195
|
+
If True, log probabilities will be used.
|
|
196
|
+
|
|
197
|
+
Returns
|
|
198
|
+
-------
|
|
199
|
+
current_state : int
|
|
200
|
+
Index of the current hidden state.
|
|
201
|
+
"""
|
|
202
|
+
self.forward_variable = self._forward_step(
|
|
203
|
+
observation_model=self.observation_model,
|
|
204
|
+
transition_model=self.transition_model,
|
|
205
|
+
observation=observation,
|
|
206
|
+
forward_variable=self.forward_variable,
|
|
207
|
+
log_probabilities=log_probabilities,
|
|
208
|
+
)
|
|
209
|
+
current_state: int = np.argmax(self.forward_variable)
|
|
210
|
+
|
|
211
|
+
return current_state
|
|
212
|
+
# pass
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
# alias
|
|
216
|
+
HMM = HiddenMarkovModel
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def viterbi_algorithm_naive(
|
|
220
|
+
hmm: HiddenMarkovModel,
|
|
221
|
+
observations: Iterable[Any],
|
|
222
|
+
log_probabilities: bool = True,
|
|
223
|
+
) -> Tuple[np.ndarray, float]:
|
|
224
|
+
"""
|
|
225
|
+
Find the most probable sequence of latent variables given
|
|
226
|
+
a sequence of observations
|
|
227
|
+
|
|
228
|
+
Parameters
|
|
229
|
+
----------
|
|
230
|
+
observations: iterable
|
|
231
|
+
An iterable containing observations. The type of each
|
|
232
|
+
element depends on input types accepted by the
|
|
233
|
+
`hmm.observation_model`
|
|
234
|
+
log_probabilities: Bool (optional)
|
|
235
|
+
If True, uses log probabilities to compute the Viterbi
|
|
236
|
+
recursion (better for numerical stability). Default is True.
|
|
237
|
+
|
|
238
|
+
Returns
|
|
239
|
+
-------
|
|
240
|
+
path: np.ndarray
|
|
241
|
+
The most probable sequence of latent variables
|
|
242
|
+
likelihood: float
|
|
243
|
+
The likelihood (either the probability or the
|
|
244
|
+
log proability if `log_probabilities` is True)
|
|
245
|
+
of the best sequence.
|
|
246
|
+
|
|
247
|
+
Note
|
|
248
|
+
----
|
|
249
|
+
This is a naïve implementation, mostly for educational purposes!
|
|
250
|
+
"""
|
|
251
|
+
# Set whether to use log probabilities in transition and
|
|
252
|
+
# observation models
|
|
253
|
+
hmm.transition_model.use_log_probabilities = log_probabilities
|
|
254
|
+
hmm.observation_model.use_log_probabilities = log_probabilities
|
|
255
|
+
# Initialize matrix for holding the best sub-sequence
|
|
256
|
+
# (log-)likelihood
|
|
257
|
+
omega = np.zeros((len(observations), hmm.n_states))
|
|
258
|
+
# Initialize dictionary for tracking the best paths
|
|
259
|
+
path = defaultdict(lambda: list())
|
|
260
|
+
|
|
261
|
+
# Initiate for i == 0
|
|
262
|
+
obs_prob = hmm.observation_model(observations[0])
|
|
263
|
+
|
|
264
|
+
if log_probabilities:
|
|
265
|
+
omega[0, :] = obs_prob + hmm.transition_model.init_probabilities
|
|
266
|
+
else:
|
|
267
|
+
omega[0, :] = obs_prob * hmm.transition_model.init_probabilities
|
|
268
|
+
|
|
269
|
+
# Viterbi recursion
|
|
270
|
+
for i, obs in enumerate(observations[1:], 1):
|
|
271
|
+
obs_prob = hmm.observation_model(obs)
|
|
272
|
+
for j in range(hmm.n_states):
|
|
273
|
+
if log_probabilities:
|
|
274
|
+
prob, state = max(
|
|
275
|
+
[
|
|
276
|
+
(omega[i - 1, k] + hmm.transition_model(k, j), k)
|
|
277
|
+
for k in range(hmm.n_states)
|
|
278
|
+
],
|
|
279
|
+
key=lambda x: x[0],
|
|
280
|
+
)
|
|
281
|
+
omega[i, j] = obs_prob[j] + prob
|
|
282
|
+
|
|
283
|
+
else:
|
|
284
|
+
prob, state = max(
|
|
285
|
+
[
|
|
286
|
+
(omega[i - 1, k] * hmm.transition_model(k, j), k)
|
|
287
|
+
for k in range(hmm.n_states)
|
|
288
|
+
],
|
|
289
|
+
key=lambda x: x[0],
|
|
290
|
+
)
|
|
291
|
+
omega[i, j] = obs_prob[j] * prob
|
|
292
|
+
# keep track of the best state
|
|
293
|
+
path[j].append(state)
|
|
294
|
+
|
|
295
|
+
# Get best path (backtracking!)
|
|
296
|
+
# Get index of the best state
|
|
297
|
+
best_sequence_idx = omega[-1, :].argmax()
|
|
298
|
+
# likelihood of the path
|
|
299
|
+
path_likelihood = omega[-1, best_sequence_idx]
|
|
300
|
+
# follow the best path backwards
|
|
301
|
+
seq = [best_sequence_idx]
|
|
302
|
+
for s in range(len(path[best_sequence_idx])):
|
|
303
|
+
best_sequence_idx = path[best_sequence_idx][-(s + 1)]
|
|
304
|
+
seq.append(best_sequence_idx)
|
|
305
|
+
# invert the path
|
|
306
|
+
best_sequence = np.array(seq[::-1], dtype=int)
|
|
307
|
+
|
|
308
|
+
if hmm.state_space is not None:
|
|
309
|
+
best_sequence = hmm.state_space[best_sequence]
|
|
310
|
+
|
|
311
|
+
return best_sequence, path_likelihood
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def viterbi_algorithm(
|
|
315
|
+
hmm: HiddenMarkovModel,
|
|
316
|
+
observations: Iterable[Any],
|
|
317
|
+
log_probabilities: bool = True,
|
|
318
|
+
) -> Tuple[np.ndarray, float]:
|
|
319
|
+
"""
|
|
320
|
+
Find the most probable sequence of latent variables given
|
|
321
|
+
a sequence of observations
|
|
322
|
+
|
|
323
|
+
Parameters
|
|
324
|
+
----------
|
|
325
|
+
observations: iterable
|
|
326
|
+
An iterable containing observations. The type of each
|
|
327
|
+
element depends on input types accepted by the
|
|
328
|
+
`hmm.observation_model`
|
|
329
|
+
log_probabilities: Bool (optional)
|
|
330
|
+
If True, uses log probabilities to compute the Viterbi
|
|
331
|
+
recursion (better for numerical stability). Default is True.
|
|
332
|
+
|
|
333
|
+
Returns
|
|
334
|
+
-------
|
|
335
|
+
path: np.ndarray
|
|
336
|
+
The most probable sequence of latent variables
|
|
337
|
+
likelihood: float
|
|
338
|
+
The likelihood (either the probability or the
|
|
339
|
+
log proability if `log_probabilities` is True)
|
|
340
|
+
of the best sequence.
|
|
341
|
+
"""
|
|
342
|
+
hmm.transition_model.use_log_probabilities = log_probabilities
|
|
343
|
+
hmm.observation_model.use_log_probabilities = log_probabilities
|
|
344
|
+
|
|
345
|
+
T = len(observations)
|
|
346
|
+
N = hmm.n_states
|
|
347
|
+
omega = np.zeros((T, N))
|
|
348
|
+
omega_idx = np.zeros((T, N), dtype=np.int64)
|
|
349
|
+
|
|
350
|
+
# Pre-compute all observation probabilities so the inner loop only
|
|
351
|
+
# operates on plain arrays (required for the Cython path).
|
|
352
|
+
obs_probs = np.ascontiguousarray(
|
|
353
|
+
[hmm.observation_model(obs) for obs in observations]
|
|
354
|
+
)
|
|
355
|
+
trans_mat = np.ascontiguousarray(hmm.transition_model())
|
|
356
|
+
init_probs = hmm.transition_model.init_probabilities
|
|
357
|
+
|
|
358
|
+
if _HAS_CYTHON:
|
|
359
|
+
_viterbi_loop(omega, omega_idx, obs_probs, trans_mat, init_probs,
|
|
360
|
+
log_probabilities)
|
|
361
|
+
elif log_probabilities:
|
|
362
|
+
omega[0, :] = obs_probs[0] + init_probs
|
|
363
|
+
for t in range(1, T):
|
|
364
|
+
prob_of_jump_to_state = omega[t - 1, :] + trans_mat.T
|
|
365
|
+
state = np.argmax(prob_of_jump_to_state, axis=1)
|
|
366
|
+
prob = prob_of_jump_to_state[np.arange(N), state]
|
|
367
|
+
omega[t, :] = obs_probs[t] + prob
|
|
368
|
+
omega_idx[t, :] = state
|
|
369
|
+
else:
|
|
370
|
+
omega[0, :] = obs_probs[0] * init_probs
|
|
371
|
+
for t in range(1, T):
|
|
372
|
+
prob_of_jump_to_state = omega[t - 1, :] * trans_mat.T
|
|
373
|
+
state = np.argmax(prob_of_jump_to_state, axis=1)
|
|
374
|
+
prob = prob_of_jump_to_state[np.arange(N), state]
|
|
375
|
+
omega[t, :] = obs_probs[t] * prob
|
|
376
|
+
omega_idx[t, :] = state
|
|
377
|
+
|
|
378
|
+
# Backtrack
|
|
379
|
+
best_sequence_idx = omega[-1, :].argmax()
|
|
380
|
+
path_likelihood = omega[-1, best_sequence_idx]
|
|
381
|
+
seq = [best_sequence_idx]
|
|
382
|
+
for s in range(T - 1):
|
|
383
|
+
best_sequence_idx = omega_idx[-(s + 1), best_sequence_idx]
|
|
384
|
+
seq.append(best_sequence_idx)
|
|
385
|
+
best_sequence = np.array(seq[::-1], dtype=int)
|
|
386
|
+
if hmm.state_space is not None:
|
|
387
|
+
best_sequence = hmm.state_space[best_sequence]
|
|
388
|
+
|
|
389
|
+
return best_sequence, path_likelihood
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
def viterbi_algorithm_windowed(
|
|
393
|
+
hmm: HiddenMarkovModel,
|
|
394
|
+
observations: Iterable[Any],
|
|
395
|
+
log_probabilities: bool = True,
|
|
396
|
+
) -> Tuple[np.ndarray, float]:
|
|
397
|
+
"""
|
|
398
|
+
Find the most probable sequence of latent variables given
|
|
399
|
+
a sequence of observations.
|
|
400
|
+
|
|
401
|
+
!!! This version uses a transition model with fixed window of
|
|
402
|
+
states (see WindowedHiddenMarkiovModel). This window represents
|
|
403
|
+
the currently "visible" states from any of an underlying
|
|
404
|
+
longer sequence of global states.
|
|
405
|
+
|
|
406
|
+
The viterbi uses the current best path at each step to
|
|
407
|
+
update the global path. This effectively limits the path
|
|
408
|
+
to corridor in the states. The observation model is called
|
|
409
|
+
with a global state id, so the true, underlying states
|
|
410
|
+
can have different probabilities.
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
Parameters
|
|
414
|
+
----------
|
|
415
|
+
observations: iterable
|
|
416
|
+
An iterable containing observations. The type of each
|
|
417
|
+
element depends on input types accepted by the
|
|
418
|
+
`hmm.observation_model`
|
|
419
|
+
log_probabilities: Bool (optional)
|
|
420
|
+
If True, uses log probabilities to compute the Viterbi
|
|
421
|
+
recursion (better for numerical stability). Default is True.
|
|
422
|
+
|
|
423
|
+
Returns
|
|
424
|
+
-------
|
|
425
|
+
path: np.ndarray
|
|
426
|
+
The most probable sequence of latent variables
|
|
427
|
+
likelihood: float
|
|
428
|
+
The likelihood (either the probability or the
|
|
429
|
+
log proability if `log_probabilities` is True)
|
|
430
|
+
of the best sequence.
|
|
431
|
+
"""
|
|
432
|
+
# Set whether to use log probabilities in transition and
|
|
433
|
+
# observation models
|
|
434
|
+
hmm.transition_model.use_log_probabilities = log_probabilities
|
|
435
|
+
hmm.observation_model.use_log_probabilities = log_probabilities
|
|
436
|
+
# Initialize matrix for holding the best sub-sequence
|
|
437
|
+
# (log-)likelihood
|
|
438
|
+
omega = np.zeros((len(observations), hmm.n_states))
|
|
439
|
+
# Initialize matrix for holding the best sub-sequence idx
|
|
440
|
+
omega_idx = np.zeros((len(observations), hmm.n_states), dtype=int)
|
|
441
|
+
|
|
442
|
+
# Initialize vector for holding the current best idx
|
|
443
|
+
# Current window id; start at state 0 and keep the state at the
|
|
444
|
+
# start of the current window
|
|
445
|
+
current_window_idx = np.zeros((len(observations) + 1), dtype=int)
|
|
446
|
+
|
|
447
|
+
omega_idx[0, :] = 0
|
|
448
|
+
current_window_idx[0] = 0 # zero is a dummy state just for jump size
|
|
449
|
+
current_window_idx[1] = 0 # zero is the actual first window idx
|
|
450
|
+
|
|
451
|
+
# Initiate for i == 0
|
|
452
|
+
obs_prob, _ = hmm.observation_model(observations[0], current_window_idx[0])
|
|
453
|
+
|
|
454
|
+
if log_probabilities:
|
|
455
|
+
omega[0, :] = obs_prob + hmm.transition_model.init_probabilities
|
|
456
|
+
else:
|
|
457
|
+
omega[0, :] = obs_prob * hmm.transition_model.init_probabilities
|
|
458
|
+
|
|
459
|
+
# Viterbi recursion
|
|
460
|
+
if log_probabilities:
|
|
461
|
+
for i, obs in enumerate(observations[1:], 1):
|
|
462
|
+
obs_prob, glob_ref_idx = hmm.observation_model(obs, current_window_idx[i])
|
|
463
|
+
# omega slice is a row vector, transition_model is a matrix
|
|
464
|
+
# of prob from state id_row to state id_column
|
|
465
|
+
# use only the slice of omega that is shifted by the previous jump
|
|
466
|
+
previous_jump = current_window_idx[i] - current_window_idx[i - 1]
|
|
467
|
+
prob_of_jump_to_state = (
|
|
468
|
+
np.concatenate(
|
|
469
|
+
(omega[i - 1, previous_jump:], np.ones(previous_jump) * -np.inf)
|
|
470
|
+
)
|
|
471
|
+
+ hmm.transition_model().T
|
|
472
|
+
)
|
|
473
|
+
state = np.argmax(prob_of_jump_to_state, axis=1)
|
|
474
|
+
prob = prob_of_jump_to_state[np.arange(hmm.n_states), state]
|
|
475
|
+
omega[i, :] = obs_prob + prob
|
|
476
|
+
omega_idx[i, :] = state
|
|
477
|
+
# slide the window to the current best
|
|
478
|
+
current_best_sequence_idx = omega[i, :].argmax()
|
|
479
|
+
current_window_idx[i + 1] = glob_ref_idx[current_best_sequence_idx]
|
|
480
|
+
|
|
481
|
+
else:
|
|
482
|
+
for i, obs in enumerate(observations[1:], 1):
|
|
483
|
+
obs_prob, glob_ref_idx = hmm.observation_model(obs, current_window_idx[i])
|
|
484
|
+
# omega slice is a row vector, transition_model is a matrix
|
|
485
|
+
# of prob from state id_row to state id_column
|
|
486
|
+
# use only the slice of omega that is shifted by the previous jump
|
|
487
|
+
previous_jump = current_window_idx[i] - current_window_idx[i - 1]
|
|
488
|
+
prob_of_jump_to_state = (
|
|
489
|
+
np.concatenate((omega[i - 1, previous_jump:], np.zeros(previous_jump)))
|
|
490
|
+
* hmm.transition_model().T
|
|
491
|
+
)
|
|
492
|
+
state = np.argmax(prob_of_jump_to_state, axis=1)
|
|
493
|
+
prob = prob_of_jump_to_state[np.arange(hmm.n_states), state]
|
|
494
|
+
omega[i, :] = obs_prob * prob
|
|
495
|
+
omega_idx[i, :] = state
|
|
496
|
+
# slide the window to the current best
|
|
497
|
+
current_best_sequence_idx = omega[i, :].argmax()
|
|
498
|
+
current_window_idx[i + 1] = glob_ref_idx[current_best_sequence_idx]
|
|
499
|
+
|
|
500
|
+
# Get best path (backtracking!)
|
|
501
|
+
# Get index of the best state
|
|
502
|
+
best_sequence_idx = omega[-1, :].argmax()
|
|
503
|
+
# likelihood of the path
|
|
504
|
+
path_likelihood = omega[-1, best_sequence_idx]
|
|
505
|
+
|
|
506
|
+
return current_window_idx[1:], path_likelihood
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
def forward_algorithm_step(
|
|
510
|
+
observation_model: ObservationModel,
|
|
511
|
+
transition_model: TransitionModel,
|
|
512
|
+
observation: Any,
|
|
513
|
+
forward_variable: Optional[np.ndarray] = None,
|
|
514
|
+
log_probabilities: bool = False,
|
|
515
|
+
) -> np.ndarray:
|
|
516
|
+
"""
|
|
517
|
+
Step of the forward algorithm.
|
|
518
|
+
|
|
519
|
+
Parameters
|
|
520
|
+
----------
|
|
521
|
+
observation_model : ObservationModel
|
|
522
|
+
An observation model.
|
|
523
|
+
transition_model: TransitionModel
|
|
524
|
+
A transition model.
|
|
525
|
+
observation: Any
|
|
526
|
+
An observation. The observation needs to be of the same
|
|
527
|
+
type as specified by the observation model.
|
|
528
|
+
forward_variable : np.ndarray
|
|
529
|
+
The forward variable of the HMM. If none is given, the
|
|
530
|
+
initial probabilities specified by the transition model
|
|
531
|
+
will be used.
|
|
532
|
+
log_probabilities : bool
|
|
533
|
+
If true, use log probabilities.
|
|
534
|
+
|
|
535
|
+
Returns
|
|
536
|
+
-------
|
|
537
|
+
forward_variable : np.ndarray
|
|
538
|
+
The updated forward variable.
|
|
539
|
+
"""
|
|
540
|
+
# since computing the log of the matrix vector multiplication is not
|
|
541
|
+
# trivial (TODO: see if there is a clever way to do this multiplication
|
|
542
|
+
# efficiently)
|
|
543
|
+
transition_model.use_log_probabilities = False
|
|
544
|
+
observation_model.use_log_probabilities = log_probabilities
|
|
545
|
+
if forward_variable is None:
|
|
546
|
+
transition_prob: np.ndarray = transition_model.init_probabilities
|
|
547
|
+
else:
|
|
548
|
+
# The matrix multiply must happen in raw-probability space.
|
|
549
|
+
# When log_probabilities=True the stored forward_variable holds log
|
|
550
|
+
# probs, so we exponentiate before the dot product.
|
|
551
|
+
fv = np.exp(forward_variable) if log_probabilities else forward_variable
|
|
552
|
+
transition_prob: np.ndarray = np.dot(transition_model().T, fv)
|
|
553
|
+
observation_prob: np.ndarray = observation_model(observation)
|
|
554
|
+
if log_probabilities:
|
|
555
|
+
forward_variable = observation_prob + np.log(transition_prob)
|
|
556
|
+
forward_variable -= logsumexp(forward_variable)
|
|
557
|
+
else:
|
|
558
|
+
forward_variable = observation_prob * transition_prob
|
|
559
|
+
forward_variable /= max(forward_variable.sum(), 1e-6)
|
|
560
|
+
|
|
561
|
+
return forward_variable
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
def forward_algorithm_step_optimized(
|
|
565
|
+
observation_model: ObservationModel,
|
|
566
|
+
transition_model: ConstantTransitionModel,
|
|
567
|
+
observation: Any,
|
|
568
|
+
forward_variable: Optional[np.ndarray] = None,
|
|
569
|
+
log_probabilities: bool = False,
|
|
570
|
+
) -> np.ndarray:
|
|
571
|
+
"""
|
|
572
|
+
Step of the forward algorithm exploiting sparsity.
|
|
573
|
+
|
|
574
|
+
Numerically equivalent to :func:`forward_algorithm_step` (up to
|
|
575
|
+
floating point summation order), but the product ``T.T @ fv`` is
|
|
576
|
+
restricted to the rows of ``T`` between the first and last state with
|
|
577
|
+
non-zero probability and to the columns that can be reached from them.
|
|
578
|
+
This is much cheaper when the forward variable is concentrated in a
|
|
579
|
+
small region of a large, banded (e.g., left-to-right) transition matrix.
|
|
580
|
+
|
|
581
|
+
Based on a proposal by Meisterschueler in
|
|
582
|
+
https://github.com/pymatchmaker/matchmaker/pull/75
|
|
583
|
+
|
|
584
|
+
Parameters
|
|
585
|
+
----------
|
|
586
|
+
observation_model : ObservationModel
|
|
587
|
+
An observation model.
|
|
588
|
+
transition_model : ConstantTransitionModel
|
|
589
|
+
A constant transition model. Other transition models are not
|
|
590
|
+
supported (use :func:`forward_algorithm_step` instead).
|
|
591
|
+
observation : Any
|
|
592
|
+
An observation. The observation needs to be of the same
|
|
593
|
+
type as specified by the observation model.
|
|
594
|
+
forward_variable : np.ndarray
|
|
595
|
+
The forward variable of the HMM. If none is given, the
|
|
596
|
+
initial probabilities specified by the transition model
|
|
597
|
+
will be used.
|
|
598
|
+
log_probabilities : bool
|
|
599
|
+
If true, use log probabilities.
|
|
600
|
+
|
|
601
|
+
Returns
|
|
602
|
+
-------
|
|
603
|
+
forward_variable : np.ndarray
|
|
604
|
+
The updated forward variable.
|
|
605
|
+
"""
|
|
606
|
+
transition_model.use_log_probabilities = False
|
|
607
|
+
observation_model.use_log_probabilities = log_probabilities
|
|
608
|
+
if forward_variable is None:
|
|
609
|
+
transition_prob: np.ndarray = transition_model.init_probabilities
|
|
610
|
+
else:
|
|
611
|
+
fv = np.exp(forward_variable) if log_probabilities else forward_variable
|
|
612
|
+
transition_prob = np.zeros_like(fv)
|
|
613
|
+
nonzero = np.flatnonzero(fv)
|
|
614
|
+
if nonzero.size:
|
|
615
|
+
lo, hi = nonzero[0], nonzero[-1] + 1
|
|
616
|
+
# Rows < hi have no non-zero entries beyond column `m - 1`.
|
|
617
|
+
m = transition_model.last_nonzero_columns[hi - 1] + 1
|
|
618
|
+
transition_prob[:m] = (
|
|
619
|
+
fv[lo:hi] @ transition_model.transition_probabilities[lo:hi, :m]
|
|
620
|
+
)
|
|
621
|
+
observation_prob: np.ndarray = observation_model(observation)
|
|
622
|
+
if log_probabilities:
|
|
623
|
+
forward_variable = observation_prob + np.log(transition_prob)
|
|
624
|
+
forward_variable -= logsumexp(forward_variable)
|
|
625
|
+
else:
|
|
626
|
+
forward_variable = observation_prob * transition_prob
|
|
627
|
+
forward_variable /= max(forward_variable.sum(), 1e-6)
|
|
628
|
+
|
|
629
|
+
return forward_variable
|
|
630
|
+
|
|
631
|
+
|
|
632
|
+
_FORWARD_ALGORITHMS: dict = {
|
|
633
|
+
"optimized": forward_algorithm_step_optimized,
|
|
634
|
+
"standard": forward_algorithm_step,
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
|
|
638
|
+
class ConstantTransitionModel(TransitionModel):
|
|
639
|
+
"""
|
|
640
|
+
Constant Transition Model
|
|
641
|
+
|
|
642
|
+
This transition model represents the case were the
|
|
643
|
+
transition proabilities do not change over time (i.e.,
|
|
644
|
+
they are static). In this case, the transition probabilities
|
|
645
|
+
can be represented by a transition matrix
|
|
646
|
+
|
|
647
|
+
Parameters
|
|
648
|
+
----------
|
|
649
|
+
transition_probabilities: np.ndarray
|
|
650
|
+
A (n_states, n_states) matrix where component
|
|
651
|
+
[i, j] represents the probability of going to state j
|
|
652
|
+
coming from state i.
|
|
653
|
+
init_probabilities: np.ndarray or None (optional)
|
|
654
|
+
A 1D vector of length n_states defining the initial
|
|
655
|
+
probabilities of each state
|
|
656
|
+
normalize_init_probabilities: bool (optional)
|
|
657
|
+
If True, the initial distribution will be normalized.
|
|
658
|
+
Default is False.
|
|
659
|
+
use_log_probabilities: bool (optional)
|
|
660
|
+
If True, use log proabilities instead of norm proabilities
|
|
661
|
+
(better for numerical stability)
|
|
662
|
+
"""
|
|
663
|
+
|
|
664
|
+
def __init__(
|
|
665
|
+
self,
|
|
666
|
+
transition_probabilities: np.ndarray,
|
|
667
|
+
init_probabilities: Optional[np.ndarray] = None,
|
|
668
|
+
normalize_init_probabilities: bool = False,
|
|
669
|
+
normalize_transition_probabilities: bool = False,
|
|
670
|
+
use_log_probabilities: bool = True,
|
|
671
|
+
) -> None:
|
|
672
|
+
super().__init__(use_log_probabilities=use_log_probabilities)
|
|
673
|
+
self.transition_probabilities = transition_probabilities
|
|
674
|
+
self.n_states = len(transition_probabilities)
|
|
675
|
+
|
|
676
|
+
if init_probabilities is None:
|
|
677
|
+
self.init_probabilities = (
|
|
678
|
+
1.0 / float(self.n_states) * np.ones(self.n_states, dtype=float)
|
|
679
|
+
)
|
|
680
|
+
else:
|
|
681
|
+
self.init_probabilities = init_probabilities
|
|
682
|
+
|
|
683
|
+
if normalize_init_probabilities:
|
|
684
|
+
# Normalize in raw-probability space, then recompute log version.
|
|
685
|
+
# (The property getter returns log probs when use_log_probabilities
|
|
686
|
+
# is True, so we bypass it and work on the backing arrays directly.)
|
|
687
|
+
self._init_dist /= np.maximum(np.sum(self._init_dist), 1e-10)
|
|
688
|
+
self._log_init_dist = np.log(self._init_dist)
|
|
689
|
+
|
|
690
|
+
if normalize_transition_probabilities:
|
|
691
|
+
self._transition_prob /= np.sum(
|
|
692
|
+
self._transition_prob, 1, keepdims=True
|
|
693
|
+
)
|
|
694
|
+
self._log_transition_prob = np.log(self._transition_prob)
|
|
695
|
+
|
|
696
|
+
@property
|
|
697
|
+
def init_probabilities(self) -> np.ndarray:
|
|
698
|
+
if self.use_log_probabilities:
|
|
699
|
+
return self._log_init_dist
|
|
700
|
+
return self._init_dist
|
|
701
|
+
|
|
702
|
+
@init_probabilities.setter
|
|
703
|
+
def init_probabilities(self, init_probabilities: np.ndarray) -> None:
|
|
704
|
+
self._init_dist = init_probabilities
|
|
705
|
+
self._log_init_dist = np.log(self._init_dist)
|
|
706
|
+
|
|
707
|
+
@property
|
|
708
|
+
def transition_probabilities(self) -> np.ndarray:
|
|
709
|
+
if self.use_log_probabilities:
|
|
710
|
+
return self._log_transition_prob
|
|
711
|
+
return self._transition_prob
|
|
712
|
+
|
|
713
|
+
@transition_probabilities.setter
|
|
714
|
+
def transition_probabilities(self, transition_probabilities: np.ndarray) -> None:
|
|
715
|
+
self._transition_prob = transition_probabilities
|
|
716
|
+
self._log_transition_prob = np.log(self._transition_prob)
|
|
717
|
+
# Rightmost non-zero column of each row, made monotonic with a
|
|
718
|
+
# cumulative max so that it bounds the columns reachable from any
|
|
719
|
+
# range of rows [0, i] (used by forward_algorithm_step_optimized).
|
|
720
|
+
T = np.asarray(transition_probabilities)
|
|
721
|
+
self._last_nonzero_columns = np.maximum.accumulate(
|
|
722
|
+
T.shape[1] - 1 - np.argmax(T[:, ::-1] != 0, axis=1)
|
|
723
|
+
)
|
|
724
|
+
|
|
725
|
+
@property
|
|
726
|
+
def last_nonzero_columns(self) -> np.ndarray:
|
|
727
|
+
return self._last_nonzero_columns
|
|
728
|
+
|
|
729
|
+
def __call__(
|
|
730
|
+
self,
|
|
731
|
+
i: Optional[int] = None,
|
|
732
|
+
j: Optional[int] = None,
|
|
733
|
+
) -> np.ndarray:
|
|
734
|
+
if i is None and j is None:
|
|
735
|
+
return self.transition_probabilities
|
|
736
|
+
if i is not None and j is None:
|
|
737
|
+
return self.transition_probabilities[i, :]
|
|
738
|
+
if i is None and j is not None:
|
|
739
|
+
return self.transition_probabilities[:, j]
|
|
740
|
+
return self.transition_probabilities[i, j]
|
|
741
|
+
|
|
742
|
+
|
|
743
|
+
class CategoricalObservationModel(ObservationModel):
|
|
744
|
+
"""
|
|
745
|
+
A Categorical observation model.
|
|
746
|
+
|
|
747
|
+
Parameters
|
|
748
|
+
----------
|
|
749
|
+
observation_probabilities : np.ndarray
|
|
750
|
+
A (num_observations, num_states) table of emission probabilities.
|
|
751
|
+
observations : Iterable or None
|
|
752
|
+
Labels for each observation row. When None, integer strings are used.
|
|
753
|
+
use_log_probabilities : bool
|
|
754
|
+
If True, use log probabilities.
|
|
755
|
+
"""
|
|
756
|
+
|
|
757
|
+
def __init__(
|
|
758
|
+
self,
|
|
759
|
+
observation_probabilities: np.ndarray,
|
|
760
|
+
observations: Optional[Iterable[Any]] = None,
|
|
761
|
+
use_log_probabilities: bool = True,
|
|
762
|
+
) -> None:
|
|
763
|
+
super().__init__(use_log_probabilities=use_log_probabilities)
|
|
764
|
+
|
|
765
|
+
self.observation_probabilities = observation_probabilities
|
|
766
|
+
|
|
767
|
+
if observations is not None:
|
|
768
|
+
self.observations: List[Any] = list(observations)
|
|
769
|
+
else:
|
|
770
|
+
self.observations = [str(i) for i in range(len(observation_probabilities))]
|
|
771
|
+
|
|
772
|
+
self.observation_indices: dict[Any, int] = {
|
|
773
|
+
obs: i for i, obs in enumerate(self.observations)
|
|
774
|
+
}
|
|
775
|
+
|
|
776
|
+
@property
|
|
777
|
+
def observation_probabilities(self) -> np.ndarray:
|
|
778
|
+
if self.use_log_probabilities:
|
|
779
|
+
return self._log_obs_prob
|
|
780
|
+
return self._obs_prob
|
|
781
|
+
|
|
782
|
+
@observation_probabilities.setter
|
|
783
|
+
def observation_probabilities(self, observation_probabilities: np.ndarray) -> None:
|
|
784
|
+
self._obs_prob = observation_probabilities
|
|
785
|
+
self._log_obs_prob = np.log(self._obs_prob)
|
|
786
|
+
|
|
787
|
+
def __call__(self, observation: Any, *args: Any, **kwargs: Any) -> np.ndarray:
|
|
788
|
+
idx = self.observation_indices[observation]
|
|
789
|
+
return self.observation_probabilities[idx]
|
|
790
|
+
|
|
791
|
+
|
|
792
|
+
# deprecated alias
|
|
793
|
+
CategoricalStringObservationModel = CategoricalObservationModel
|
|
794
|
+
|
|
795
|
+
|
|
796
|
+
class WindowedObservationModel(ObservationModel):
|
|
797
|
+
"""
|
|
798
|
+
Windowed Observation Model.
|
|
799
|
+
|
|
800
|
+
Uses a list of state-specific probability models as the observation model.
|
|
801
|
+
Distances are supported: pass ``use_inverted_probs=True`` to convert
|
|
802
|
+
distances (lower = more probable) to probabilities via inverted softmax.
|
|
803
|
+
|
|
804
|
+
Only works with :func:`viterbi_algorithm_windowed`.
|
|
805
|
+
|
|
806
|
+
Parameters
|
|
807
|
+
----------
|
|
808
|
+
prob_models_at_state : list of lists of (callable, int) pairs
|
|
809
|
+
``prob_models_at_state[state_id]`` is a list of
|
|
810
|
+
``(probability_model, global_state_index)`` pairs for the window
|
|
811
|
+
centred at ``state_id``.
|
|
812
|
+
use_inverted_probs : bool
|
|
813
|
+
When True, model outputs are treated as distances and converted to
|
|
814
|
+
probabilities via inverted softmax (lower distance → higher prob).
|
|
815
|
+
When False, outputs are treated as scores and converted via softmax.
|
|
816
|
+
use_log_probabilities : bool
|
|
817
|
+
If True, return log probabilities.
|
|
818
|
+
"""
|
|
819
|
+
|
|
820
|
+
def __init__(
|
|
821
|
+
self,
|
|
822
|
+
prob_models_at_state: List[List[_ProbModelEntry]],
|
|
823
|
+
use_inverted_probs: bool = True,
|
|
824
|
+
use_log_probabilities: bool = True,
|
|
825
|
+
) -> None:
|
|
826
|
+
super().__init__(use_log_probabilities=use_log_probabilities)
|
|
827
|
+
self.use_inverted_probs: bool = use_inverted_probs
|
|
828
|
+
self.prob_models_at_state: List[List[_ProbModelEntry]] = prob_models_at_state
|
|
829
|
+
self.state_number: int = len(prob_models_at_state)
|
|
830
|
+
|
|
831
|
+
def __call__(
|
|
832
|
+
self,
|
|
833
|
+
observation: Any,
|
|
834
|
+
current_state: int,
|
|
835
|
+
*args: Any,
|
|
836
|
+
**kwargs: Any,
|
|
837
|
+
) -> Tuple[np.ndarray, np.ndarray]:
|
|
838
|
+
prob_models = self.prob_models_at_state[
|
|
839
|
+
max(0, min(current_state, self.state_number - 1))
|
|
840
|
+
]
|
|
841
|
+
dists: List[Any] = []
|
|
842
|
+
glob_ref_idx: List[int] = []
|
|
843
|
+
for prob_model, idx in prob_models:
|
|
844
|
+
dists.append(prob_model(observation))
|
|
845
|
+
glob_ref_idx.append(idx)
|
|
846
|
+
|
|
847
|
+
arr = np.array(dists)
|
|
848
|
+
probs = inverted_softmax(arr) if self.use_inverted_probs else softmax(arr)
|
|
849
|
+
|
|
850
|
+
if self.use_log_probabilities:
|
|
851
|
+
return np.log(probs), np.array(glob_ref_idx)
|
|
852
|
+
return probs, np.array(glob_ref_idx)
|