txtures 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- txtures-0.1.0/LICENSE +21 -0
- txtures-0.1.0/PKG-INFO +109 -0
- txtures-0.1.0/README.md +76 -0
- txtures-0.1.0/pyproject.toml +78 -0
- txtures-0.1.0/setup.cfg +4 -0
- txtures-0.1.0/src/txtures/__init__.py +26 -0
- txtures-0.1.0/src/txtures/__main__.py +7 -0
- txtures-0.1.0/src/txtures/api.py +254 -0
- txtures-0.1.0/src/txtures/cli.py +728 -0
- txtures-0.1.0/src/txtures/corpus.py +95 -0
- txtures-0.1.0/src/txtures/definitions/__init__.py +22 -0
- txtures-0.1.0/src/txtures/definitions/resolution.py +577 -0
- txtures-0.1.0/src/txtures/definitions/store.py +223 -0
- txtures-0.1.0/src/txtures/engine/__init__.py +27 -0
- txtures-0.1.0/src/txtures/engine/execution.py +280 -0
- txtures-0.1.0/src/txtures/engine/planning.py +240 -0
- txtures-0.1.0/src/txtures/engine/slurm.py +214 -0
- txtures-0.1.0/src/txtures/engine/tables.py +385 -0
- txtures-0.1.0/src/txtures/environment.py +79 -0
- txtures-0.1.0/src/txtures/errors.py +44 -0
- txtures-0.1.0/src/txtures/identity.py +25 -0
- txtures-0.1.0/src/txtures/inputs.py +628 -0
- txtures-0.1.0/src/txtures/installation.py +239 -0
- txtures-0.1.0/src/txtures/modules/__init__.py +9 -0
- txtures-0.1.0/src/txtures/modules/base.py +250 -0
- txtures-0.1.0/src/txtures/modules/hf_model.py +345 -0
- txtures-0.1.0/src/txtures/modules/lm_scores.py +269 -0
- txtures-0.1.0/src/txtures/modules/surprisal.py +113 -0
- txtures-0.1.0/src/txtures/settings.py +412 -0
- txtures-0.1.0/src/txtures/store.py +305 -0
- txtures-0.1.0/src/txtures/yamlio.py +55 -0
- txtures-0.1.0/src/txtures.egg-info/PKG-INFO +109 -0
- txtures-0.1.0/src/txtures.egg-info/SOURCES.txt +47 -0
- txtures-0.1.0/src/txtures.egg-info/dependency_links.txt +1 -0
- txtures-0.1.0/src/txtures.egg-info/entry_points.txt +2 -0
- txtures-0.1.0/src/txtures.egg-info/requires.txt +12 -0
- txtures-0.1.0/src/txtures.egg-info/top_level.txt +1 -0
- txtures-0.1.0/tests/test_api.py +173 -0
- txtures-0.1.0/tests/test_cli.py +220 -0
- txtures-0.1.0/tests/test_engine.py +379 -0
- txtures-0.1.0/tests/test_inputs.py +326 -0
- txtures-0.1.0/tests/test_installation.py +143 -0
- txtures-0.1.0/tests/test_lm.py +486 -0
- txtures-0.1.0/tests/test_lm_integration.py +241 -0
- txtures-0.1.0/tests/test_resolution.py +327 -0
- txtures-0.1.0/tests/test_settings.py +162 -0
- txtures-0.1.0/tests/test_slurm.py +276 -0
- txtures-0.1.0/tests/test_store.py +95 -0
- txtures-0.1.0/tests/test_yamlio.py +26 -0
txtures-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 txtures contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
txtures-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: txtures
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Word- and region-level features of text for psycholinguistic research
|
|
5
|
+
Author: Cory Shain
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Documentation, https://txtures.readthedocs.io
|
|
8
|
+
Project-URL: Repository, https://github.com/CLiMBlab-Stanford/txtures
|
|
9
|
+
Project-URL: Issues, https://github.com/CLiMBlab-Stanford/txtures/issues
|
|
10
|
+
Keywords: psycholinguistics,surprisal,language models,reading times
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Operating System :: MacOS
|
|
14
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Topic :: Scientific/Engineering
|
|
18
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
19
|
+
Requires-Python: >=3.12
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
Requires-Dist: huggingface-hub<3,>=1
|
|
23
|
+
Requires-Dist: numpy<3,>=1.26
|
|
24
|
+
Requires-Dist: pandas<3,>=2.2
|
|
25
|
+
Requires-Dist: pydantic<3,>=2.7
|
|
26
|
+
Requires-Dist: PyYAML<7,>=6
|
|
27
|
+
Requires-Dist: transformers<6,>=5
|
|
28
|
+
Provides-Extra: cuda
|
|
29
|
+
Requires-Dist: torch<3,>=2.4; extra == "cuda"
|
|
30
|
+
Provides-Extra: cpu
|
|
31
|
+
Requires-Dist: torch<3,>=2.4; extra == "cpu"
|
|
32
|
+
Dynamic: license-file
|
|
33
|
+
|
|
34
|
+
# txtures
|
|
35
|
+
|
|
36
|
+
txtures computes word- and region-level features of text for psycholinguistic
|
|
37
|
+
research. Its first feature is surprisal from causal language models on the
|
|
38
|
+
Hugging Face hub, with word-boundary correction and user-defined regions.
|
|
39
|
+
|
|
40
|
+
txtures is in early development. Its interfaces will change during the 0.x
|
|
41
|
+
series.
|
|
42
|
+
|
|
43
|
+
## Installation
|
|
44
|
+
|
|
45
|
+
From a checkout on Linux or macOS:
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
./install
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
This downloads a pinned uv and Python and installs txtures with its locked
|
|
52
|
+
dependencies into `txtures-install/` in the checkout, without using or
|
|
53
|
+
changing any Python already on the machine. It puts a `txtures` command in
|
|
54
|
+
`~/.local/bin`. Useful options:
|
|
55
|
+
|
|
56
|
+
- `--compute cpu` installs PyTorch without CUDA (the default on Linux is CUDA).
|
|
57
|
+
- `--site DIR` uses a lab's definitions store, whose site settings name the
|
|
58
|
+
shared stores of models and results. `txtures site use DIR` does the same
|
|
59
|
+
later.
|
|
60
|
+
- `--prefix DIR` installs somewhere other than the checkout.
|
|
61
|
+
|
|
62
|
+
On a Slurm cluster, `./install --executor slurm` makes `txtures run` submit
|
|
63
|
+
its computation as a job and return at once; set the partition with
|
|
64
|
+
`txtures site set slurm.partition NAME` (and optionally `slurm.account`,
|
|
65
|
+
`slurm.gpus`, and others), and follow the job with `txtures status`.
|
|
66
|
+
|
|
67
|
+
Run `./install --help` for every option and `txtures site ls` to see the
|
|
68
|
+
resulting settings. For gated models, run `txtures hf login` once.
|
|
69
|
+
|
|
70
|
+
txtures can also be installed with pip or uv as a dependency of other
|
|
71
|
+
software; see
|
|
72
|
+
[Settings](https://txtures.readthedocs.io/en/latest/settings/#installation).
|
|
73
|
+
|
|
74
|
+
## Use
|
|
75
|
+
|
|
76
|
+
Put a file of sentences, one per line, in the inputs directory as
|
|
77
|
+
`gardenpath.txt`, then:
|
|
78
|
+
|
|
79
|
+
```bash
|
|
80
|
+
txtures run gardenpath
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
This writes a table with one row per word and its surprisal under GPT-2 to
|
|
84
|
+
the derivatives directory. From Python:
|
|
85
|
+
|
|
86
|
+
```python
|
|
87
|
+
import txtures as tx
|
|
88
|
+
|
|
89
|
+
table = tx.compute("surprisal", ["The horse raced past the barn fell."])
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
See the [documentation](https://txtures.readthedocs.io), starting with
|
|
93
|
+
[Getting started](https://txtures.readthedocs.io/en/latest/getting-started/).
|
|
94
|
+
|
|
95
|
+
## Development
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
./install --dev
|
|
99
|
+
txtures-install/env/bin/python -m pytest -q
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
See [CONTRIBUTING.md](https://github.com/CLiMBlab-Stanford/txtures/blob/main/CONTRIBUTING.md)
|
|
103
|
+
for contribution, attribution, and release rules, and
|
|
104
|
+
[WRITING_POLICY.md](https://github.com/CLiMBlab-Stanford/txtures/blob/main/WRITING_POLICY.md)
|
|
105
|
+
for documentation style.
|
|
106
|
+
|
|
107
|
+
## License
|
|
108
|
+
|
|
109
|
+
MIT. See [LICENSE](https://github.com/CLiMBlab-Stanford/txtures/blob/main/LICENSE).
|
txtures-0.1.0/README.md
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# txtures
|
|
2
|
+
|
|
3
|
+
txtures computes word- and region-level features of text for psycholinguistic
|
|
4
|
+
research. Its first feature is surprisal from causal language models on the
|
|
5
|
+
Hugging Face hub, with word-boundary correction and user-defined regions.
|
|
6
|
+
|
|
7
|
+
txtures is in early development. Its interfaces will change during the 0.x
|
|
8
|
+
series.
|
|
9
|
+
|
|
10
|
+
## Installation
|
|
11
|
+
|
|
12
|
+
From a checkout on Linux or macOS:
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
./install
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
This downloads a pinned uv and Python and installs txtures with its locked
|
|
19
|
+
dependencies into `txtures-install/` in the checkout, without using or
|
|
20
|
+
changing any Python already on the machine. It puts a `txtures` command in
|
|
21
|
+
`~/.local/bin`. Useful options:
|
|
22
|
+
|
|
23
|
+
- `--compute cpu` installs PyTorch without CUDA (the default on Linux is CUDA).
|
|
24
|
+
- `--site DIR` uses a lab's definitions store, whose site settings name the
|
|
25
|
+
shared stores of models and results. `txtures site use DIR` does the same
|
|
26
|
+
later.
|
|
27
|
+
- `--prefix DIR` installs somewhere other than the checkout.
|
|
28
|
+
|
|
29
|
+
On a Slurm cluster, `./install --executor slurm` makes `txtures run` submit
|
|
30
|
+
its computation as a job and return at once; set the partition with
|
|
31
|
+
`txtures site set slurm.partition NAME` (and optionally `slurm.account`,
|
|
32
|
+
`slurm.gpus`, and others), and follow the job with `txtures status`.
|
|
33
|
+
|
|
34
|
+
Run `./install --help` for every option and `txtures site ls` to see the
|
|
35
|
+
resulting settings. For gated models, run `txtures hf login` once.
|
|
36
|
+
|
|
37
|
+
txtures can also be installed with pip or uv as a dependency of other
|
|
38
|
+
software; see
|
|
39
|
+
[Settings](https://txtures.readthedocs.io/en/latest/settings/#installation).
|
|
40
|
+
|
|
41
|
+
## Use
|
|
42
|
+
|
|
43
|
+
Put a file of sentences, one per line, in the inputs directory as
|
|
44
|
+
`gardenpath.txt`, then:
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
txtures run gardenpath
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
This writes a table with one row per word and its surprisal under GPT-2 to
|
|
51
|
+
the derivatives directory. From Python:
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
import txtures as tx
|
|
55
|
+
|
|
56
|
+
table = tx.compute("surprisal", ["The horse raced past the barn fell."])
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
See the [documentation](https://txtures.readthedocs.io), starting with
|
|
60
|
+
[Getting started](https://txtures.readthedocs.io/en/latest/getting-started/).
|
|
61
|
+
|
|
62
|
+
## Development
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
./install --dev
|
|
66
|
+
txtures-install/env/bin/python -m pytest -q
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
See [CONTRIBUTING.md](https://github.com/CLiMBlab-Stanford/txtures/blob/main/CONTRIBUTING.md)
|
|
70
|
+
for contribution, attribution, and release rules, and
|
|
71
|
+
[WRITING_POLICY.md](https://github.com/CLiMBlab-Stanford/txtures/blob/main/WRITING_POLICY.md)
|
|
72
|
+
for documentation style.
|
|
73
|
+
|
|
74
|
+
## License
|
|
75
|
+
|
|
76
|
+
MIT. See [LICENSE](https://github.com/CLiMBlab-Stanford/txtures/blob/main/LICENSE).
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=64"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "txtures"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Word- and region-level features of text for psycholinguistic research"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
requires-python = ">=3.12"
|
|
13
|
+
authors = [{ name = "Cory Shain" }]
|
|
14
|
+
keywords = ["psycholinguistics", "surprisal", "language models", "reading times"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 3 - Alpha",
|
|
17
|
+
"Intended Audience :: Science/Research",
|
|
18
|
+
"Operating System :: MacOS",
|
|
19
|
+
"Operating System :: POSIX :: Linux",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.12",
|
|
22
|
+
"Topic :: Scientific/Engineering",
|
|
23
|
+
"Topic :: Text Processing :: Linguistic",
|
|
24
|
+
]
|
|
25
|
+
dependencies = [
|
|
26
|
+
"huggingface-hub>=1,<3",
|
|
27
|
+
"numpy>=1.26,<3",
|
|
28
|
+
"pandas>=2.2,<3",
|
|
29
|
+
"pydantic>=2.7,<3",
|
|
30
|
+
"PyYAML>=6,<7",
|
|
31
|
+
"transformers>=5,<6",
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
[project.optional-dependencies]
|
|
35
|
+
# Exactly one of these selects the PyTorch build. `./install --compute`
|
|
36
|
+
# chooses between them.
|
|
37
|
+
cuda = ["torch>=2.4,<3"]
|
|
38
|
+
cpu = ["torch>=2.4,<3"]
|
|
39
|
+
|
|
40
|
+
[dependency-groups]
|
|
41
|
+
dev = ["pytest>=8,<10", "ruff>=0.12,<1"]
|
|
42
|
+
|
|
43
|
+
[project.urls]
|
|
44
|
+
Documentation = "https://txtures.readthedocs.io"
|
|
45
|
+
Repository = "https://github.com/CLiMBlab-Stanford/txtures"
|
|
46
|
+
Issues = "https://github.com/CLiMBlab-Stanford/txtures/issues"
|
|
47
|
+
|
|
48
|
+
[project.scripts]
|
|
49
|
+
txtures = "txtures.cli:main"
|
|
50
|
+
|
|
51
|
+
[tool.setuptools.packages.find]
|
|
52
|
+
where = ["src"]
|
|
53
|
+
|
|
54
|
+
[tool.pytest.ini_options]
|
|
55
|
+
testpaths = ["tests"]
|
|
56
|
+
addopts = ["-m", "not integration"]
|
|
57
|
+
markers = [
|
|
58
|
+
"integration: downloads models or exercises process and filesystem boundaries",
|
|
59
|
+
]
|
|
60
|
+
|
|
61
|
+
[tool.ruff]
|
|
62
|
+
line-length = 100
|
|
63
|
+
target-version = "py312"
|
|
64
|
+
extend-exclude = ["notebooks"]
|
|
65
|
+
|
|
66
|
+
[tool.ruff.lint]
|
|
67
|
+
select = ["E4", "E7", "E9", "F", "I", "B", "UP"]
|
|
68
|
+
|
|
69
|
+
[tool.uv]
|
|
70
|
+
conflicts = [[{ extra = "cuda" }, { extra = "cpu" }]]
|
|
71
|
+
|
|
72
|
+
[tool.uv.sources]
|
|
73
|
+
torch = [{ index = "pytorch-cpu", extra = "cpu" }]
|
|
74
|
+
|
|
75
|
+
[[tool.uv.index]]
|
|
76
|
+
name = "pytorch-cpu"
|
|
77
|
+
url = "https://download.pytorch.org/whl/cpu"
|
|
78
|
+
explicit = true
|
txtures-0.1.0/setup.cfg
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""Word- and region-level features of text for psycholinguistic research.
|
|
2
|
+
|
|
3
|
+
The Python API::
|
|
4
|
+
|
|
5
|
+
import txtures as tx
|
|
6
|
+
|
|
7
|
+
table = tx.compute("surprisal", ["The horse raced past the barn fell."])
|
|
8
|
+
table = tx.run("bk21")
|
|
9
|
+
table = tx.read_table("derivatives/bk21/main-0123456789ab")
|
|
10
|
+
|
|
11
|
+
See `txtures.api` for the details.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from importlib.metadata import version
|
|
15
|
+
|
|
16
|
+
__version__ = version("txtures")
|
|
17
|
+
__all__ = ["compute", "read_table", "run"]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def __getattr__(name: str):
|
|
21
|
+
# Import the API on first use, so the command line starts without pandas.
|
|
22
|
+
if name in __all__:
|
|
23
|
+
from txtures import api
|
|
24
|
+
|
|
25
|
+
return getattr(api, name)
|
|
26
|
+
raise AttributeError(f"module 'txtures' has no attribute {name!r}")
|
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
"""The Python API: the same requests as the command line, from Python.
|
|
2
|
+
|
|
3
|
+
`run` is `txtures run`: it computes a named input's request and writes the
|
|
4
|
+
same output table, then returns the table. `compute` computes features of
|
|
5
|
+
text given directly and returns the table without writing one. Both use the
|
|
6
|
+
configured item and resource stores, so repeated calls reuse earlier work.
|
|
7
|
+
|
|
8
|
+
The API always computes in the calling process, whatever
|
|
9
|
+
`execution.executor` says; submit Slurm jobs with `txtures run`.
|
|
10
|
+
|
|
11
|
+
The API does not change the process environment. It passes its Hugging
|
|
12
|
+
Face cache and token to the Hugging Face libraries directly, so a host
|
|
13
|
+
program's own use of those libraries is unaffected.
|
|
14
|
+
|
|
15
|
+
When the definitions store, item store, and resources all come from
|
|
16
|
+
built-in defaults, the first call warns once and suggests how to point
|
|
17
|
+
txtures at shared locations.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import warnings
|
|
23
|
+
from collections.abc import Callable, Mapping, Sequence
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
from typing import TYPE_CHECKING, Any
|
|
26
|
+
|
|
27
|
+
from txtures import environment, installation
|
|
28
|
+
from txtures.definitions import Definitions
|
|
29
|
+
from txtures.definitions.resolution import EXTENDS, MAIN
|
|
30
|
+
from txtures.definitions.store import Definition, StoreContents
|
|
31
|
+
from txtures.engine import Executor, Plan, plan_request, run_request
|
|
32
|
+
from txtures.engine.tables import read_table, table_frame
|
|
33
|
+
|
|
34
|
+
__all__ = ["compute", "read_table", "run"]
|
|
35
|
+
from txtures.errors import DefinitionError, InputError, Problem
|
|
36
|
+
from txtures.inputs import InputSettings, read_texts
|
|
37
|
+
from txtures.modules import REGISTRY
|
|
38
|
+
from txtures.settings import Settings, resolve_settings, uses_default_locations
|
|
39
|
+
from txtures.store import EntryStore
|
|
40
|
+
|
|
41
|
+
if TYPE_CHECKING:
|
|
42
|
+
import pandas as pd
|
|
43
|
+
|
|
44
|
+
Texts = str | Sequence[str | Sequence[str]] | Mapping[str, str | Sequence[str]]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
DEFAULT_LOCATIONS_MESSAGE = """\
|
|
48
|
+
txtures is using its default locations in your home directory, so it shares no
|
|
49
|
+
results with others. To use a shared site, run once:
|
|
50
|
+
|
|
51
|
+
txtures site use /path/to/txtures-definitions
|
|
52
|
+
|
|
53
|
+
or set locations one at a time:
|
|
54
|
+
|
|
55
|
+
txtures site set items /path/to/items
|
|
56
|
+
txtures site set resources /path/to/txtures-resources
|
|
57
|
+
|
|
58
|
+
`txtures site ls` lists every setting. Environment variables such as
|
|
59
|
+
TXTURES_DEFINITIONS, and arguments to the Python API, also set locations."""
|
|
60
|
+
|
|
61
|
+
_warned_defaults = False
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def settings(flags: Mapping[str, Any] | None = None) -> Settings:
|
|
65
|
+
"""Resolve settings, with `flags` taking precedence.
|
|
66
|
+
|
|
67
|
+
`flags` maps setting keys, such as `items` or `execution.device`, to
|
|
68
|
+
values. The installation running this interpreter, if any, supplies the
|
|
69
|
+
settings it records. Warns once per process when every shared location
|
|
70
|
+
comes from built-in defaults.
|
|
71
|
+
|
|
72
|
+
Raises:
|
|
73
|
+
SettingsError: A settings document or value is invalid.
|
|
74
|
+
"""
|
|
75
|
+
global _warned_defaults
|
|
76
|
+
installed = installation.current()
|
|
77
|
+
resolved = resolve_settings(
|
|
78
|
+
flags,
|
|
79
|
+
installation=installed.settings if installed else None,
|
|
80
|
+
installation_source=installed.source if installed else "installation",
|
|
81
|
+
)
|
|
82
|
+
if not _warned_defaults and uses_default_locations(resolved):
|
|
83
|
+
_warned_defaults = True
|
|
84
|
+
warnings.warn(DEFAULT_LOCATIONS_MESSAGE, DefaultLocationsWarning, stacklevel=3)
|
|
85
|
+
return resolved
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class DefaultLocationsWarning(UserWarning):
|
|
89
|
+
"""Every shared location comes from built-in defaults."""
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def load_definitions(resolved: Settings) -> Definitions:
|
|
93
|
+
"""Load the selected definitions store, or the packaged definitions if it does not exist."""
|
|
94
|
+
if resolved.definitions.is_dir():
|
|
95
|
+
return Definitions.load(resolved.definitions)
|
|
96
|
+
return Definitions.packaged()
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def local_executor(plan: Plan, resolved: Settings, progress: Callable[[str], None] | None = None):
|
|
100
|
+
"""An executor for `plan` on this machine, using the configured stores."""
|
|
101
|
+
return Executor(
|
|
102
|
+
plan,
|
|
103
|
+
EntryStore(resolved.items),
|
|
104
|
+
EntryStore(resolved.resource_store),
|
|
105
|
+
environment.run_context(resolved),
|
|
106
|
+
progress,
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _flags(locations: Mapping[str, Any], device: str | None) -> dict[str, Any]:
|
|
111
|
+
flags = {key: str(value) for key, value in locations.items() if value is not None}
|
|
112
|
+
if device is not None:
|
|
113
|
+
flags["execution.device"] = device
|
|
114
|
+
return flags
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def run(
|
|
118
|
+
name: str,
|
|
119
|
+
workflow: str = MAIN,
|
|
120
|
+
modules: Sequence[str] | None = None,
|
|
121
|
+
*,
|
|
122
|
+
definitions: Path | str | None = None,
|
|
123
|
+
inputs: Path | str | None = None,
|
|
124
|
+
derivatives: Path | str | None = None,
|
|
125
|
+
items: Path | str | None = None,
|
|
126
|
+
resources: Path | str | None = None,
|
|
127
|
+
device: str | None = None,
|
|
128
|
+
progress: Callable[[str], None] | None = None,
|
|
129
|
+
) -> pd.DataFrame:
|
|
130
|
+
"""Compute a request for the input `name` and return its output table.
|
|
131
|
+
|
|
132
|
+
This is `txtures run NAME --workflow WORKFLOW --modules ...`: the table
|
|
133
|
+
is written to the derivatives directory, or reused if it is current, and
|
|
134
|
+
then read with `read_table`. The keyword arguments override settings for
|
|
135
|
+
this call; `progress`, if given, receives progress messages.
|
|
136
|
+
|
|
137
|
+
Raises:
|
|
138
|
+
TxturesError: The input, definitions, or settings are invalid, or a
|
|
139
|
+
computation failed.
|
|
140
|
+
"""
|
|
141
|
+
resolved = settings(
|
|
142
|
+
_flags(
|
|
143
|
+
{
|
|
144
|
+
"definitions": definitions,
|
|
145
|
+
"inputs": inputs,
|
|
146
|
+
"derivatives": derivatives,
|
|
147
|
+
"items": items,
|
|
148
|
+
"resources": resources,
|
|
149
|
+
},
|
|
150
|
+
device,
|
|
151
|
+
)
|
|
152
|
+
)
|
|
153
|
+
plan = plan_request(resolved, load_definitions(resolved), name, workflow, modules)
|
|
154
|
+
_warn(plan.notes)
|
|
155
|
+
directory, _ = run_request(plan, local_executor(plan, resolved, progress), modules)
|
|
156
|
+
return read_table(directory)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def compute(
|
|
160
|
+
modules: str | Sequence[str],
|
|
161
|
+
texts: Texts,
|
|
162
|
+
*,
|
|
163
|
+
config: Mapping[str, Mapping[str, Any]] | None = None,
|
|
164
|
+
workflow: str | None = None,
|
|
165
|
+
region_delimiter: str | None = None,
|
|
166
|
+
sentence_delimiter: str | None = None,
|
|
167
|
+
region_joiner: str = " ",
|
|
168
|
+
sentence_joiner: str = " ",
|
|
169
|
+
definitions: Path | str | None = None,
|
|
170
|
+
items: Path | str | None = None,
|
|
171
|
+
resources: Path | str | None = None,
|
|
172
|
+
device: str | None = None,
|
|
173
|
+
progress: Callable[[str], None] | None = None,
|
|
174
|
+
) -> pd.DataFrame:
|
|
175
|
+
"""Compute the features of `modules` for text given directly, and return the table.
|
|
176
|
+
|
|
177
|
+
`texts` is one document, a sequence of documents (with IDs `1`, `2`,
|
|
178
|
+
...), or a mapping from document ID to document. Each document is a
|
|
179
|
+
string, divided into regions and sentences by the delimiters as an input
|
|
180
|
+
file's sidecar would divide it, or a sequence of region strings.
|
|
181
|
+
|
|
182
|
+
Configs come from one of two places:
|
|
183
|
+
|
|
184
|
+
- `config` maps module names to config bodies, such as
|
|
185
|
+
`{"lm_scores": {"model": "gpt2"}}`. Each body overrides the packaged
|
|
186
|
+
defaults, may sweep parameters as a config file can, and ignores the
|
|
187
|
+
definitions store, so the call alone determines the result. Modules
|
|
188
|
+
not named use their packaged defaults.
|
|
189
|
+
- Otherwise the definitions store's `workflow` (default `main`) chooses
|
|
190
|
+
them, as for `txtures run`.
|
|
191
|
+
|
|
192
|
+
The table has the columns of an output table. No table is written, but
|
|
193
|
+
computed items are stored and reused.
|
|
194
|
+
|
|
195
|
+
Raises:
|
|
196
|
+
TxturesError: The text, configs, or settings are invalid, or a
|
|
197
|
+
computation failed.
|
|
198
|
+
"""
|
|
199
|
+
if config is not None and workflow is not None:
|
|
200
|
+
raise DefinitionError([Problem("compute", "give either config or workflow, not both")])
|
|
201
|
+
requested = [modules] if isinstance(modules, str) else list(modules)
|
|
202
|
+
report = read_texts(
|
|
203
|
+
texts,
|
|
204
|
+
InputSettings(
|
|
205
|
+
region_delimiter=region_delimiter,
|
|
206
|
+
sentence_delimiter=sentence_delimiter,
|
|
207
|
+
region_joiner=region_joiner,
|
|
208
|
+
sentence_joiner=sentence_joiner,
|
|
209
|
+
),
|
|
210
|
+
)
|
|
211
|
+
if report.corpus is None:
|
|
212
|
+
raise InputError(report.problems)
|
|
213
|
+
_warn(str(warning) for warning in report.warnings)
|
|
214
|
+
resolved = settings(
|
|
215
|
+
_flags({"definitions": definitions, "items": items, "resources": resources}, device)
|
|
216
|
+
)
|
|
217
|
+
chosen = load_definitions(resolved) if config is None else _given(config)
|
|
218
|
+
plan = plan_request(
|
|
219
|
+
resolved,
|
|
220
|
+
chosen,
|
|
221
|
+
report.corpus.name,
|
|
222
|
+
workflow or MAIN,
|
|
223
|
+
requested,
|
|
224
|
+
corpus=report.corpus,
|
|
225
|
+
)
|
|
226
|
+
_warn(plan.notes)
|
|
227
|
+
run_executor = local_executor(plan, resolved, progress)
|
|
228
|
+
run_executor.prepare_resources()
|
|
229
|
+
run_executor.compute_items()
|
|
230
|
+
return table_frame(plan, run_executor)
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _given(config: Mapping[str, Mapping[str, Any]]) -> Definitions:
|
|
234
|
+
"""Packaged definitions whose `main` configs are the bodies given to `compute`."""
|
|
235
|
+
contents = StoreContents(None)
|
|
236
|
+
problems = []
|
|
237
|
+
for module_name, body in config.items():
|
|
238
|
+
where = f"config[{module_name!r}]"
|
|
239
|
+
if module_name not in REGISTRY:
|
|
240
|
+
problems.append(Problem(where, f"unknown module {module_name!r}"))
|
|
241
|
+
elif not isinstance(body, Mapping):
|
|
242
|
+
problems.append(Problem(where, "must be a mapping of config fields"))
|
|
243
|
+
elif EXTENDS in body:
|
|
244
|
+
problems.append(Problem(where, "extends is not available here"))
|
|
245
|
+
else:
|
|
246
|
+
contents.configs[module_name] = {MAIN: Definition(MAIN, dict(body), None, source=where)}
|
|
247
|
+
if problems:
|
|
248
|
+
raise DefinitionError(problems)
|
|
249
|
+
return Definitions(contents, REGISTRY)
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _warn(messages) -> None:
|
|
253
|
+
for message in messages:
|
|
254
|
+
warnings.warn(message, stacklevel=3)
|