txtures 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- txtures/__init__.py +26 -0
- txtures/__main__.py +7 -0
- txtures/api.py +254 -0
- txtures/cli.py +728 -0
- txtures/corpus.py +95 -0
- txtures/definitions/__init__.py +22 -0
- txtures/definitions/resolution.py +577 -0
- txtures/definitions/store.py +223 -0
- txtures/engine/__init__.py +27 -0
- txtures/engine/execution.py +280 -0
- txtures/engine/planning.py +240 -0
- txtures/engine/slurm.py +214 -0
- txtures/engine/tables.py +385 -0
- txtures/environment.py +79 -0
- txtures/errors.py +44 -0
- txtures/identity.py +25 -0
- txtures/inputs.py +628 -0
- txtures/installation.py +239 -0
- txtures/modules/__init__.py +9 -0
- txtures/modules/base.py +250 -0
- txtures/modules/hf_model.py +345 -0
- txtures/modules/lm_scores.py +269 -0
- txtures/modules/surprisal.py +113 -0
- txtures/settings.py +412 -0
- txtures/store.py +305 -0
- txtures/yamlio.py +55 -0
- txtures-0.1.0.dist-info/METADATA +109 -0
- txtures-0.1.0.dist-info/RECORD +32 -0
- txtures-0.1.0.dist-info/WHEEL +5 -0
- txtures-0.1.0.dist-info/entry_points.txt +2 -0
- txtures-0.1.0.dist-info/licenses/LICENSE +21 -0
- txtures-0.1.0.dist-info/top_level.txt +1 -0
txtures/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""Word- and region-level features of text for psycholinguistic research.
|
|
2
|
+
|
|
3
|
+
The Python API::
|
|
4
|
+
|
|
5
|
+
import txtures as tx
|
|
6
|
+
|
|
7
|
+
table = tx.compute("surprisal", ["The horse raced past the barn fell."])
|
|
8
|
+
table = tx.run("bk21")
|
|
9
|
+
table = tx.read_table("derivatives/bk21/main-0123456789ab")
|
|
10
|
+
|
|
11
|
+
See `txtures.api` for the details.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from importlib.metadata import version
|
|
15
|
+
|
|
16
|
+
__version__ = version("txtures")
|
|
17
|
+
__all__ = ["compute", "read_table", "run"]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def __getattr__(name: str):
|
|
21
|
+
# Import the API on first use, so the command line starts without pandas.
|
|
22
|
+
if name in __all__:
|
|
23
|
+
from txtures import api
|
|
24
|
+
|
|
25
|
+
return getattr(api, name)
|
|
26
|
+
raise AttributeError(f"module 'txtures' has no attribute {name!r}")
|
txtures/__main__.py
ADDED
txtures/api.py
ADDED
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
"""The Python API: the same requests as the command line, from Python.
|
|
2
|
+
|
|
3
|
+
`run` is `txtures run`: it computes a named input's request and writes the
|
|
4
|
+
same output table, then returns the table. `compute` computes features of
|
|
5
|
+
text given directly and returns the table without writing one. Both use the
|
|
6
|
+
configured item and resource stores, so repeated calls reuse earlier work.
|
|
7
|
+
|
|
8
|
+
The API always computes in the calling process, whatever
|
|
9
|
+
`execution.executor` says; submit Slurm jobs with `txtures run`.
|
|
10
|
+
|
|
11
|
+
The API does not change the process environment. It passes its Hugging
|
|
12
|
+
Face cache and token to the Hugging Face libraries directly, so a host
|
|
13
|
+
program's own use of those libraries is unaffected.
|
|
14
|
+
|
|
15
|
+
When the definitions store, item store, and resources all come from
|
|
16
|
+
built-in defaults, the first call warns once and suggests how to point
|
|
17
|
+
txtures at shared locations.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import warnings
|
|
23
|
+
from collections.abc import Callable, Mapping, Sequence
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
from typing import TYPE_CHECKING, Any
|
|
26
|
+
|
|
27
|
+
from txtures import environment, installation
|
|
28
|
+
from txtures.definitions import Definitions
|
|
29
|
+
from txtures.definitions.resolution import EXTENDS, MAIN
|
|
30
|
+
from txtures.definitions.store import Definition, StoreContents
|
|
31
|
+
from txtures.engine import Executor, Plan, plan_request, run_request
|
|
32
|
+
from txtures.engine.tables import read_table, table_frame
|
|
33
|
+
|
|
34
|
+
__all__ = ["compute", "read_table", "run"]
|
|
35
|
+
from txtures.errors import DefinitionError, InputError, Problem
|
|
36
|
+
from txtures.inputs import InputSettings, read_texts
|
|
37
|
+
from txtures.modules import REGISTRY
|
|
38
|
+
from txtures.settings import Settings, resolve_settings, uses_default_locations
|
|
39
|
+
from txtures.store import EntryStore
|
|
40
|
+
|
|
41
|
+
if TYPE_CHECKING:
|
|
42
|
+
import pandas as pd
|
|
43
|
+
|
|
44
|
+
Texts = str | Sequence[str | Sequence[str]] | Mapping[str, str | Sequence[str]]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
DEFAULT_LOCATIONS_MESSAGE = """\
|
|
48
|
+
txtures is using its default locations in your home directory, so it shares no
|
|
49
|
+
results with others. To use a shared site, run once:
|
|
50
|
+
|
|
51
|
+
txtures site use /path/to/txtures-definitions
|
|
52
|
+
|
|
53
|
+
or set locations one at a time:
|
|
54
|
+
|
|
55
|
+
txtures site set items /path/to/items
|
|
56
|
+
txtures site set resources /path/to/txtures-resources
|
|
57
|
+
|
|
58
|
+
`txtures site ls` lists every setting. Environment variables such as
|
|
59
|
+
TXTURES_DEFINITIONS, and arguments to the Python API, also set locations."""
|
|
60
|
+
|
|
61
|
+
_warned_defaults = False
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def settings(flags: Mapping[str, Any] | None = None) -> Settings:
|
|
65
|
+
"""Resolve settings, with `flags` taking precedence.
|
|
66
|
+
|
|
67
|
+
`flags` maps setting keys, such as `items` or `execution.device`, to
|
|
68
|
+
values. The installation running this interpreter, if any, supplies the
|
|
69
|
+
settings it records. Warns once per process when every shared location
|
|
70
|
+
comes from built-in defaults.
|
|
71
|
+
|
|
72
|
+
Raises:
|
|
73
|
+
SettingsError: A settings document or value is invalid.
|
|
74
|
+
"""
|
|
75
|
+
global _warned_defaults
|
|
76
|
+
installed = installation.current()
|
|
77
|
+
resolved = resolve_settings(
|
|
78
|
+
flags,
|
|
79
|
+
installation=installed.settings if installed else None,
|
|
80
|
+
installation_source=installed.source if installed else "installation",
|
|
81
|
+
)
|
|
82
|
+
if not _warned_defaults and uses_default_locations(resolved):
|
|
83
|
+
_warned_defaults = True
|
|
84
|
+
warnings.warn(DEFAULT_LOCATIONS_MESSAGE, DefaultLocationsWarning, stacklevel=3)
|
|
85
|
+
return resolved
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class DefaultLocationsWarning(UserWarning):
|
|
89
|
+
"""Every shared location comes from built-in defaults."""
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def load_definitions(resolved: Settings) -> Definitions:
|
|
93
|
+
"""Load the selected definitions store, or the packaged definitions if it does not exist."""
|
|
94
|
+
if resolved.definitions.is_dir():
|
|
95
|
+
return Definitions.load(resolved.definitions)
|
|
96
|
+
return Definitions.packaged()
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def local_executor(plan: Plan, resolved: Settings, progress: Callable[[str], None] | None = None):
|
|
100
|
+
"""An executor for `plan` on this machine, using the configured stores."""
|
|
101
|
+
return Executor(
|
|
102
|
+
plan,
|
|
103
|
+
EntryStore(resolved.items),
|
|
104
|
+
EntryStore(resolved.resource_store),
|
|
105
|
+
environment.run_context(resolved),
|
|
106
|
+
progress,
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _flags(locations: Mapping[str, Any], device: str | None) -> dict[str, Any]:
|
|
111
|
+
flags = {key: str(value) for key, value in locations.items() if value is not None}
|
|
112
|
+
if device is not None:
|
|
113
|
+
flags["execution.device"] = device
|
|
114
|
+
return flags
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def run(
|
|
118
|
+
name: str,
|
|
119
|
+
workflow: str = MAIN,
|
|
120
|
+
modules: Sequence[str] | None = None,
|
|
121
|
+
*,
|
|
122
|
+
definitions: Path | str | None = None,
|
|
123
|
+
inputs: Path | str | None = None,
|
|
124
|
+
derivatives: Path | str | None = None,
|
|
125
|
+
items: Path | str | None = None,
|
|
126
|
+
resources: Path | str | None = None,
|
|
127
|
+
device: str | None = None,
|
|
128
|
+
progress: Callable[[str], None] | None = None,
|
|
129
|
+
) -> pd.DataFrame:
|
|
130
|
+
"""Compute a request for the input `name` and return its output table.
|
|
131
|
+
|
|
132
|
+
This is `txtures run NAME --workflow WORKFLOW --modules ...`: the table
|
|
133
|
+
is written to the derivatives directory, or reused if it is current, and
|
|
134
|
+
then read with `read_table`. The keyword arguments override settings for
|
|
135
|
+
this call; `progress`, if given, receives progress messages.
|
|
136
|
+
|
|
137
|
+
Raises:
|
|
138
|
+
TxturesError: The input, definitions, or settings are invalid, or a
|
|
139
|
+
computation failed.
|
|
140
|
+
"""
|
|
141
|
+
resolved = settings(
|
|
142
|
+
_flags(
|
|
143
|
+
{
|
|
144
|
+
"definitions": definitions,
|
|
145
|
+
"inputs": inputs,
|
|
146
|
+
"derivatives": derivatives,
|
|
147
|
+
"items": items,
|
|
148
|
+
"resources": resources,
|
|
149
|
+
},
|
|
150
|
+
device,
|
|
151
|
+
)
|
|
152
|
+
)
|
|
153
|
+
plan = plan_request(resolved, load_definitions(resolved), name, workflow, modules)
|
|
154
|
+
_warn(plan.notes)
|
|
155
|
+
directory, _ = run_request(plan, local_executor(plan, resolved, progress), modules)
|
|
156
|
+
return read_table(directory)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def compute(
|
|
160
|
+
modules: str | Sequence[str],
|
|
161
|
+
texts: Texts,
|
|
162
|
+
*,
|
|
163
|
+
config: Mapping[str, Mapping[str, Any]] | None = None,
|
|
164
|
+
workflow: str | None = None,
|
|
165
|
+
region_delimiter: str | None = None,
|
|
166
|
+
sentence_delimiter: str | None = None,
|
|
167
|
+
region_joiner: str = " ",
|
|
168
|
+
sentence_joiner: str = " ",
|
|
169
|
+
definitions: Path | str | None = None,
|
|
170
|
+
items: Path | str | None = None,
|
|
171
|
+
resources: Path | str | None = None,
|
|
172
|
+
device: str | None = None,
|
|
173
|
+
progress: Callable[[str], None] | None = None,
|
|
174
|
+
) -> pd.DataFrame:
|
|
175
|
+
"""Compute the features of `modules` for text given directly, and return the table.
|
|
176
|
+
|
|
177
|
+
`texts` is one document, a sequence of documents (with IDs `1`, `2`,
|
|
178
|
+
...), or a mapping from document ID to document. Each document is a
|
|
179
|
+
string, divided into regions and sentences by the delimiters as an input
|
|
180
|
+
file's sidecar would divide it, or a sequence of region strings.
|
|
181
|
+
|
|
182
|
+
Configs come from one of two places:
|
|
183
|
+
|
|
184
|
+
- `config` maps module names to config bodies, such as
|
|
185
|
+
`{"lm_scores": {"model": "gpt2"}}`. Each body overrides the packaged
|
|
186
|
+
defaults, may sweep parameters as a config file can, and ignores the
|
|
187
|
+
definitions store, so the call alone determines the result. Modules
|
|
188
|
+
not named use their packaged defaults.
|
|
189
|
+
- Otherwise the definitions store's `workflow` (default `main`) chooses
|
|
190
|
+
them, as for `txtures run`.
|
|
191
|
+
|
|
192
|
+
The table has the columns of an output table. No table is written, but
|
|
193
|
+
computed items are stored and reused.
|
|
194
|
+
|
|
195
|
+
Raises:
|
|
196
|
+
TxturesError: The text, configs, or settings are invalid, or a
|
|
197
|
+
computation failed.
|
|
198
|
+
"""
|
|
199
|
+
if config is not None and workflow is not None:
|
|
200
|
+
raise DefinitionError([Problem("compute", "give either config or workflow, not both")])
|
|
201
|
+
requested = [modules] if isinstance(modules, str) else list(modules)
|
|
202
|
+
report = read_texts(
|
|
203
|
+
texts,
|
|
204
|
+
InputSettings(
|
|
205
|
+
region_delimiter=region_delimiter,
|
|
206
|
+
sentence_delimiter=sentence_delimiter,
|
|
207
|
+
region_joiner=region_joiner,
|
|
208
|
+
sentence_joiner=sentence_joiner,
|
|
209
|
+
),
|
|
210
|
+
)
|
|
211
|
+
if report.corpus is None:
|
|
212
|
+
raise InputError(report.problems)
|
|
213
|
+
_warn(str(warning) for warning in report.warnings)
|
|
214
|
+
resolved = settings(
|
|
215
|
+
_flags({"definitions": definitions, "items": items, "resources": resources}, device)
|
|
216
|
+
)
|
|
217
|
+
chosen = load_definitions(resolved) if config is None else _given(config)
|
|
218
|
+
plan = plan_request(
|
|
219
|
+
resolved,
|
|
220
|
+
chosen,
|
|
221
|
+
report.corpus.name,
|
|
222
|
+
workflow or MAIN,
|
|
223
|
+
requested,
|
|
224
|
+
corpus=report.corpus,
|
|
225
|
+
)
|
|
226
|
+
_warn(plan.notes)
|
|
227
|
+
run_executor = local_executor(plan, resolved, progress)
|
|
228
|
+
run_executor.prepare_resources()
|
|
229
|
+
run_executor.compute_items()
|
|
230
|
+
return table_frame(plan, run_executor)
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _given(config: Mapping[str, Mapping[str, Any]]) -> Definitions:
|
|
234
|
+
"""Packaged definitions whose `main` configs are the bodies given to `compute`."""
|
|
235
|
+
contents = StoreContents(None)
|
|
236
|
+
problems = []
|
|
237
|
+
for module_name, body in config.items():
|
|
238
|
+
where = f"config[{module_name!r}]"
|
|
239
|
+
if module_name not in REGISTRY:
|
|
240
|
+
problems.append(Problem(where, f"unknown module {module_name!r}"))
|
|
241
|
+
elif not isinstance(body, Mapping):
|
|
242
|
+
problems.append(Problem(where, "must be a mapping of config fields"))
|
|
243
|
+
elif EXTENDS in body:
|
|
244
|
+
problems.append(Problem(where, "extends is not available here"))
|
|
245
|
+
else:
|
|
246
|
+
contents.configs[module_name] = {MAIN: Definition(MAIN, dict(body), None, source=where)}
|
|
247
|
+
if problems:
|
|
248
|
+
raise DefinitionError(problems)
|
|
249
|
+
return Definitions(contents, REGISTRY)
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _warn(messages) -> None:
|
|
253
|
+
for message in messages:
|
|
254
|
+
warnings.warn(message, stacklevel=3)
|