txtures 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
txtures/__init__.py ADDED
@@ -0,0 +1,26 @@
1
+ """Word- and region-level features of text for psycholinguistic research.
2
+
3
+ The Python API::
4
+
5
+ import txtures as tx
6
+
7
+ table = tx.compute("surprisal", ["The horse raced past the barn fell."])
8
+ table = tx.run("bk21")
9
+ table = tx.read_table("derivatives/bk21/main-0123456789ab")
10
+
11
+ See `txtures.api` for the details.
12
+ """
13
+
14
+ from importlib.metadata import version
15
+
16
+ __version__ = version("txtures")
17
+ __all__ = ["compute", "read_table", "run"]
18
+
19
+
20
+ def __getattr__(name: str):
21
+ # Import the API on first use, so the command line starts without pandas.
22
+ if name in __all__:
23
+ from txtures import api
24
+
25
+ return getattr(api, name)
26
+ raise AttributeError(f"module 'txtures' has no attribute {name!r}")
txtures/__main__.py ADDED
@@ -0,0 +1,7 @@
1
+ """Run the txtures command line with `python -m txtures`."""
2
+
3
+ import sys
4
+
5
+ from txtures.cli import main
6
+
7
+ sys.exit(main())
txtures/api.py ADDED
@@ -0,0 +1,254 @@
1
+ """The Python API: the same requests as the command line, from Python.
2
+
3
+ `run` is `txtures run`: it computes a named input's request and writes the
4
+ same output table, then returns the table. `compute` computes features of
5
+ text given directly and returns the table without writing one. Both use the
6
+ configured item and resource stores, so repeated calls reuse earlier work.
7
+
8
+ The API always computes in the calling process, whatever
9
+ `execution.executor` says; submit Slurm jobs with `txtures run`.
10
+
11
+ The API does not change the process environment. It passes its Hugging
12
+ Face cache and token to the Hugging Face libraries directly, so a host
13
+ program's own use of those libraries is unaffected.
14
+
15
+ When the definitions store, item store, and resources all come from
16
+ built-in defaults, the first call warns once and suggests how to point
17
+ txtures at shared locations.
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ import warnings
23
+ from collections.abc import Callable, Mapping, Sequence
24
+ from pathlib import Path
25
+ from typing import TYPE_CHECKING, Any
26
+
27
+ from txtures import environment, installation
28
+ from txtures.definitions import Definitions
29
+ from txtures.definitions.resolution import EXTENDS, MAIN
30
+ from txtures.definitions.store import Definition, StoreContents
31
+ from txtures.engine import Executor, Plan, plan_request, run_request
32
+ from txtures.engine.tables import read_table, table_frame
33
+
34
+ __all__ = ["compute", "read_table", "run"]
35
+ from txtures.errors import DefinitionError, InputError, Problem
36
+ from txtures.inputs import InputSettings, read_texts
37
+ from txtures.modules import REGISTRY
38
+ from txtures.settings import Settings, resolve_settings, uses_default_locations
39
+ from txtures.store import EntryStore
40
+
41
+ if TYPE_CHECKING:
42
+ import pandas as pd
43
+
44
+ Texts = str | Sequence[str | Sequence[str]] | Mapping[str, str | Sequence[str]]
45
+
46
+
47
+ DEFAULT_LOCATIONS_MESSAGE = """\
48
+ txtures is using its default locations in your home directory, so it shares no
49
+ results with others. To use a shared site, run once:
50
+
51
+ txtures site use /path/to/txtures-definitions
52
+
53
+ or set locations one at a time:
54
+
55
+ txtures site set items /path/to/items
56
+ txtures site set resources /path/to/txtures-resources
57
+
58
+ `txtures site ls` lists every setting. Environment variables such as
59
+ TXTURES_DEFINITIONS, and arguments to the Python API, also set locations."""
60
+
61
+ _warned_defaults = False
62
+
63
+
64
+ def settings(flags: Mapping[str, Any] | None = None) -> Settings:
65
+ """Resolve settings, with `flags` taking precedence.
66
+
67
+ `flags` maps setting keys, such as `items` or `execution.device`, to
68
+ values. The installation running this interpreter, if any, supplies the
69
+ settings it records. Warns once per process when every shared location
70
+ comes from built-in defaults.
71
+
72
+ Raises:
73
+ SettingsError: A settings document or value is invalid.
74
+ """
75
+ global _warned_defaults
76
+ installed = installation.current()
77
+ resolved = resolve_settings(
78
+ flags,
79
+ installation=installed.settings if installed else None,
80
+ installation_source=installed.source if installed else "installation",
81
+ )
82
+ if not _warned_defaults and uses_default_locations(resolved):
83
+ _warned_defaults = True
84
+ warnings.warn(DEFAULT_LOCATIONS_MESSAGE, DefaultLocationsWarning, stacklevel=3)
85
+ return resolved
86
+
87
+
88
+ class DefaultLocationsWarning(UserWarning):
89
+ """Every shared location comes from built-in defaults."""
90
+
91
+
92
+ def load_definitions(resolved: Settings) -> Definitions:
93
+ """Load the selected definitions store, or the packaged definitions if it does not exist."""
94
+ if resolved.definitions.is_dir():
95
+ return Definitions.load(resolved.definitions)
96
+ return Definitions.packaged()
97
+
98
+
99
+ def local_executor(plan: Plan, resolved: Settings, progress: Callable[[str], None] | None = None):
100
+ """An executor for `plan` on this machine, using the configured stores."""
101
+ return Executor(
102
+ plan,
103
+ EntryStore(resolved.items),
104
+ EntryStore(resolved.resource_store),
105
+ environment.run_context(resolved),
106
+ progress,
107
+ )
108
+
109
+
110
+ def _flags(locations: Mapping[str, Any], device: str | None) -> dict[str, Any]:
111
+ flags = {key: str(value) for key, value in locations.items() if value is not None}
112
+ if device is not None:
113
+ flags["execution.device"] = device
114
+ return flags
115
+
116
+
117
+ def run(
118
+ name: str,
119
+ workflow: str = MAIN,
120
+ modules: Sequence[str] | None = None,
121
+ *,
122
+ definitions: Path | str | None = None,
123
+ inputs: Path | str | None = None,
124
+ derivatives: Path | str | None = None,
125
+ items: Path | str | None = None,
126
+ resources: Path | str | None = None,
127
+ device: str | None = None,
128
+ progress: Callable[[str], None] | None = None,
129
+ ) -> pd.DataFrame:
130
+ """Compute a request for the input `name` and return its output table.
131
+
132
+ This is `txtures run NAME --workflow WORKFLOW --modules ...`: the table
133
+ is written to the derivatives directory, or reused if it is current, and
134
+ then read with `read_table`. The keyword arguments override settings for
135
+ this call; `progress`, if given, receives progress messages.
136
+
137
+ Raises:
138
+ TxturesError: The input, definitions, or settings are invalid, or a
139
+ computation failed.
140
+ """
141
+ resolved = settings(
142
+ _flags(
143
+ {
144
+ "definitions": definitions,
145
+ "inputs": inputs,
146
+ "derivatives": derivatives,
147
+ "items": items,
148
+ "resources": resources,
149
+ },
150
+ device,
151
+ )
152
+ )
153
+ plan = plan_request(resolved, load_definitions(resolved), name, workflow, modules)
154
+ _warn(plan.notes)
155
+ directory, _ = run_request(plan, local_executor(plan, resolved, progress), modules)
156
+ return read_table(directory)
157
+
158
+
159
+ def compute(
160
+ modules: str | Sequence[str],
161
+ texts: Texts,
162
+ *,
163
+ config: Mapping[str, Mapping[str, Any]] | None = None,
164
+ workflow: str | None = None,
165
+ region_delimiter: str | None = None,
166
+ sentence_delimiter: str | None = None,
167
+ region_joiner: str = " ",
168
+ sentence_joiner: str = " ",
169
+ definitions: Path | str | None = None,
170
+ items: Path | str | None = None,
171
+ resources: Path | str | None = None,
172
+ device: str | None = None,
173
+ progress: Callable[[str], None] | None = None,
174
+ ) -> pd.DataFrame:
175
+ """Compute the features of `modules` for text given directly, and return the table.
176
+
177
+ `texts` is one document, a sequence of documents (with IDs `1`, `2`,
178
+ ...), or a mapping from document ID to document. Each document is a
179
+ string, divided into regions and sentences by the delimiters as an input
180
+ file's sidecar would divide it, or a sequence of region strings.
181
+
182
+ Configs come from one of two places:
183
+
184
+ - `config` maps module names to config bodies, such as
185
+ `{"lm_scores": {"model": "gpt2"}}`. Each body overrides the packaged
186
+ defaults, may sweep parameters as a config file can, and ignores the
187
+ definitions store, so the call alone determines the result. Modules
188
+ not named use their packaged defaults.
189
+ - Otherwise the definitions store's `workflow` (default `main`) chooses
190
+ them, as for `txtures run`.
191
+
192
+ The table has the columns of an output table. No table is written, but
193
+ computed items are stored and reused.
194
+
195
+ Raises:
196
+ TxturesError: The text, configs, or settings are invalid, or a
197
+ computation failed.
198
+ """
199
+ if config is not None and workflow is not None:
200
+ raise DefinitionError([Problem("compute", "give either config or workflow, not both")])
201
+ requested = [modules] if isinstance(modules, str) else list(modules)
202
+ report = read_texts(
203
+ texts,
204
+ InputSettings(
205
+ region_delimiter=region_delimiter,
206
+ sentence_delimiter=sentence_delimiter,
207
+ region_joiner=region_joiner,
208
+ sentence_joiner=sentence_joiner,
209
+ ),
210
+ )
211
+ if report.corpus is None:
212
+ raise InputError(report.problems)
213
+ _warn(str(warning) for warning in report.warnings)
214
+ resolved = settings(
215
+ _flags({"definitions": definitions, "items": items, "resources": resources}, device)
216
+ )
217
+ chosen = load_definitions(resolved) if config is None else _given(config)
218
+ plan = plan_request(
219
+ resolved,
220
+ chosen,
221
+ report.corpus.name,
222
+ workflow or MAIN,
223
+ requested,
224
+ corpus=report.corpus,
225
+ )
226
+ _warn(plan.notes)
227
+ run_executor = local_executor(plan, resolved, progress)
228
+ run_executor.prepare_resources()
229
+ run_executor.compute_items()
230
+ return table_frame(plan, run_executor)
231
+
232
+
233
+ def _given(config: Mapping[str, Mapping[str, Any]]) -> Definitions:
234
+ """Packaged definitions whose `main` configs are the bodies given to `compute`."""
235
+ contents = StoreContents(None)
236
+ problems = []
237
+ for module_name, body in config.items():
238
+ where = f"config[{module_name!r}]"
239
+ if module_name not in REGISTRY:
240
+ problems.append(Problem(where, f"unknown module {module_name!r}"))
241
+ elif not isinstance(body, Mapping):
242
+ problems.append(Problem(where, "must be a mapping of config fields"))
243
+ elif EXTENDS in body:
244
+ problems.append(Problem(where, "extends is not available here"))
245
+ else:
246
+ contents.configs[module_name] = {MAIN: Definition(MAIN, dict(body), None, source=where)}
247
+ if problems:
248
+ raise DefinitionError(problems)
249
+ return Definitions(contents, REGISTRY)
250
+
251
+
252
+ def _warn(messages) -> None:
253
+ for message in messages:
254
+ warnings.warn(message, stacklevel=3)